diff --git a/vendor/github.com/RoaringBitmap/roaring/AUTHORS b/vendor/github.com/RoaringBitmap/roaring/AUTHORS new file mode 100644 index 00000000..08c07404 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/AUTHORS @@ -0,0 +1,10 @@ +# This is the official list of roaring authors for copyright purposes. + +Todd Gruben (@tgruben), +Daniel Lemire (@lemire), +Elliot Murphy (@statik), +Bob Potter (@bpot), +Tyson Maly (@tvmaly), +Will Glynn (@willglynn), +Brent Pedersen (@brentp) +Maciej Biłas (@maciej) diff --git a/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS new file mode 100644 index 00000000..70b4735d --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS @@ -0,0 +1,12 @@ +# This is the official list of roaring contributors + +Todd Gruben (@tgruben), +Daniel Lemire (@lemire), +Elliot Murphy (@statik), +Bob Potter (@bpot), +Tyson Maly (@tvmaly), +Will Glynn (@willglynn), +Brent Pedersen (@brentp), +Jason E. Aten (@glycerine), +Vali Malinoiu (@0x4139), +Forud Ghafouri (@fzerorubigd) \ No newline at end of file diff --git a/vendor/github.com/RoaringBitmap/roaring/LICENSE b/vendor/github.com/RoaringBitmap/roaring/LICENSE new file mode 100644 index 00000000..aff5f999 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 by the authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt b/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt new file mode 100644 index 00000000..aff5f999 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/LICENSE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 by the authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/RoaringBitmap/roaring/Makefile b/vendor/github.com/RoaringBitmap/roaring/Makefile new file mode 100644 index 00000000..d5259b4c --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/Makefile @@ -0,0 +1,121 @@ +.PHONY: help all test format fmtcheck vet lint qa deps clean nuke rle backrle ser fetch-real-roaring-datasets + + + + + + + + +# Display general help about this command +help: + @echo "" + @echo "The following commands are available:" + @echo "" + @echo " make qa : Run all the tests" + @echo " make test : Run the unit tests" + @echo "" + @echo " make format : Format the source code" + @echo " make fmtcheck : Check if the source code has been formatted" + @echo " make vet : Check for suspicious constructs" + @echo " make lint : Check for style errors" + @echo "" + @echo " make deps : Get the dependencies" + @echo " make clean : Remove any build artifact" + @echo " make nuke : Deletes any intermediate file" + @echo "" + @echo " make fuzz-smat : Fuzzy testing with smat" + @echo " make fuzz-stream : Fuzzy testing with stream deserialization" + @echo " make fuzz-buffer : Fuzzy testing with buffer deserialization" + @echo "" + +# Alias for help target +all: help +test: + go test + go test -race -run TestConcurrent* +# Format the source code +format: + @find ./ -type f -name "*.go" -exec gofmt -w {} \; + +# Check if the source code has been formatted +fmtcheck: + @mkdir -p target + @find ./ -type f -name "*.go" -exec gofmt -d {} \; | tee target/format.diff + @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; } + +# Check for syntax errors +vet: + GOPATH=$(GOPATH) go vet ./... + +# Check for style errors +lint: + GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./... + + + + + +# Alias to run all quality-assurance checks +qa: fmtcheck test vet lint + +# --- INSTALL --- + +# Get the dependencies +deps: + GOPATH=$(GOPATH) go get github.com/smartystreets/goconvey/convey + GOPATH=$(GOPATH) go get github.com/willf/bitset + GOPATH=$(GOPATH) go get github.com/golang/lint/golint + GOPATH=$(GOPATH) go get github.com/mschoch/smat + GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz + GOPATH=$(GOPATH) go get github.com/dvyukov/go-fuzz/go-fuzz-build + GOPATH=$(GOPATH) go get github.com/glycerine/go-unsnap-stream + GOPATH=$(GOPATH) go get github.com/philhofer/fwd + GOPATH=$(GOPATH) go get github.com/jtolds/gls + +fuzz-smat: + go test -tags=gofuzz -run=TestGenerateSmatCorpus + go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring + go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 + + +fuzz-stream: + go-fuzz-build -func FuzzSerializationStream github.com/RoaringBitmap/roaring + go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 + + +fuzz-buffer: + go-fuzz-build -func FuzzSerializationBuffer github.com/RoaringBitmap/roaring + go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 + +# Remove any build artifact +clean: + GOPATH=$(GOPATH) go clean ./... + +# Deletes any intermediate file +nuke: + rm -rf ./target + GOPATH=$(GOPATH) go clean -i ./... + +rle: + cp rle.go rle16.go + perl -pi -e 's/32/16/g' rle16.go + cp rle_test.go rle16_test.go + perl -pi -e 's/32/16/g' rle16_test.go + +backrle: + cp rle16.go rle.go + perl -pi -e 's/16/32/g' rle.go + perl -pi -e 's/2032/2016/g' rle.go + +ser: rle + go generate + +cover: + go test -coverprofile=coverage.out + go tool cover -html=coverage.out + +fetch-real-roaring-datasets: + # pull github.com/RoaringBitmap/real-roaring-datasets -> testdata/real-roaring-datasets + git submodule init + git submodule update diff --git a/vendor/github.com/RoaringBitmap/roaring/README.md b/vendor/github.com/RoaringBitmap/roaring/README.md new file mode 100644 index 00000000..2c096ce8 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/README.md @@ -0,0 +1,246 @@ +roaring [![Build Status](https://travis-ci.org/RoaringBitmap/roaring.png)](https://travis-ci.org/RoaringBitmap/roaring) [![Coverage Status](https://coveralls.io/repos/github/RoaringBitmap/roaring/badge.svg?branch=master)](https://coveralls.io/github/RoaringBitmap/roaring?branch=master) [![GoDoc](https://godoc.org/github.com/RoaringBitmap/roaring?status.svg)](https://godoc.org/github.com/RoaringBitmap/roaring) [![Go Report Card](https://goreportcard.com/badge/RoaringBitmap/roaring)](https://goreportcard.com/report/github.com/RoaringBitmap/roaring) +============= + +This is a go version of the Roaring bitmap data structure. + + + +Roaring bitmaps are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and +[Elasticsearch][elasticsearch], [Metamarkets' Druid][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [Pilosa][pilosa], [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. + +[lucene]: https://lucene.apache.org/ +[solr]: https://lucene.apache.org/solr/ +[elasticsearch]: https://www.elastic.co/products/elasticsearch +[druid]: http://druid.io/ +[spark]: https://spark.apache.org/ +[opensearchserver]: http://www.opensearchserver.com +[cloudtorrent]: https://github.com/jpillora/cloud-torrent +[whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home +[pilosa]: https://www.pilosa.com/ +[kylin]: http://kylin.apache.org/ +[pinot]: http://github.com/linkedin/pinot/wiki +[vsts]: https://www.visualstudio.com/team-services/ +[atlas]: https://github.com/Netflix/atlas + +Roaring bitmaps are found to work well in many important applications: + +> Use Roaring for bitmap compression whenever possible. Do not use other bitmap compression methods ([Wang et al., SIGMOD 2017](http://db.ucsd.edu/wp-content/uploads/2017/03/sidm338-wangA.pdf)) + + +The ``roaring`` Go library is used by +* [Cloud Torrent](https://github.com/jpillora/cloud-torrent): a self-hosted remote torrent client +* [runv](https://github.com/hyperhq/runv): an Hypervisor-based runtime for the Open Containers Initiative +* [InfluxDB](https://www.influxdata.com) +* [Pilosa](https://www.pilosa.com/) +* [Bleve](http://www.blevesearch.com) + +This library is used in production in several systems, it is part of the [Awesome Go collection](https://awesome-go.com). + + +There are also [Java](https://github.com/RoaringBitmap/RoaringBitmap) and [C/C++](https://github.com/RoaringBitmap/CRoaring) versions. The Java, C, C++ and Go version are binary compatible: e.g, you can save bitmaps +from a Java program and load them back in Go, and vice versa. We have a [format specification](https://github.com/RoaringBitmap/RoaringFormatSpec). + + +This code is licensed under Apache License, Version 2.0 (ASL2.0). + +Copyright 2016-... by the authors. + + +### References + +- Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821) +- Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin, +Better bitmap performance with Roaring bitmaps, +Software: Practice and Experience 46 (5), 2016. +http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/realroaring2014.html +- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience 46 (11), 2016. http://arxiv.org/abs/1603.06549 + + +### Dependencies + +Dependencies are fetched automatically by giving the `-t` flag to `go get`. + +they include + - github.com/smartystreets/goconvey/convey + - github.com/willf/bitset + - github.com/mschoch/smat + - github.com/glycerine/go-unsnap-stream + - github.com/philhofer/fwd + - github.com/jtolds/gls + +Note that the smat library requires Go 1.6 or better. + +#### Installation + + - go get -t github.com/RoaringBitmap/roaring + + +### Example + +Here is a simplified but complete example: + +```go +package main + +import ( + "fmt" + "github.com/RoaringBitmap/roaring" + "bytes" +) + + +func main() { + // example inspired by https://github.com/fzandona/goroar + fmt.Println("==roaring==") + rb1 := roaring.BitmapOf(1, 2, 3, 4, 5, 100, 1000) + fmt.Println(rb1.String()) + + rb2 := roaring.BitmapOf(3, 4, 1000) + fmt.Println(rb2.String()) + + rb3 := roaring.New() + fmt.Println(rb3.String()) + + fmt.Println("Cardinality: ", rb1.GetCardinality()) + + fmt.Println("Contains 3? ", rb1.Contains(3)) + + rb1.And(rb2) + + rb3.Add(1) + rb3.Add(5) + + rb3.Or(rb1) + + // computes union of the three bitmaps in parallel using 4 workers + roaring.ParOr(4, rb1, rb2, rb3) + // computes intersection of the three bitmaps in parallel using 4 workers + roaring.ParAnd(4, rb1, rb2, rb3) + + + // prints 1, 3, 4, 5, 1000 + i := rb3.Iterator() + for i.HasNext() { + fmt.Println(i.Next()) + } + fmt.Println() + + // next we include an example of serialization + buf := new(bytes.Buffer) + rb1.WriteTo(buf) // we omit error handling + newrb:= roaring.New() + newrb.ReadFrom(buf) + if rb1.Equals(newrb) { + fmt.Println("I wrote the content to a byte stream and read it back.") + } +} +``` + +If you wish to use serialization and handle errors, you might want to +consider the following sample of code: + +```go + rb := BitmapOf(1, 2, 3, 4, 5, 100, 1000) + buf := new(bytes.Buffer) + size,err:=rb.WriteTo(buf) + if err != nil { + t.Errorf("Failed writing") + } + newrb:= New() + size,err=newrb.ReadFrom(buf) + if err != nil { + t.Errorf("Failed reading") + } + if ! rb.Equals(newrb) { + t.Errorf("Cannot retrieve serialized version") + } +``` + +Given N integers in [0,x), then the serialized size in bytes of +a Roaring bitmap should never exceed this bound: + +`` 8 + 9 * ((long)x+65535)/65536 + 2 * N `` + +That is, given a fixed overhead for the universe size (x), Roaring +bitmaps never use more than 2 bytes per integer. You can call +``BoundSerializedSizeInBytes`` for a more precise estimate. + + +### Documentation + +Current documentation is available at http://godoc.org/github.com/RoaringBitmap/roaring + +### Goroutine safety + +In general, it should not generally be considered safe to access +the same bitmaps using different goroutines--they are left +unsynchronized for performance. Should you want to access +a Bitmap from more than one goroutine, you should +provide synchronization. Typically this is done by using channels to pass +the *Bitmap around (in Go style; so there is only ever one owner), +or by using `sync.Mutex` to serialize operations on Bitmaps. + +### Coverage + +We test our software. For a report on our test coverage, see + +https://coveralls.io/github/RoaringBitmap/roaring?branch=master + +### Benchmark + +Type + + go test -bench Benchmark -run - + +To run benchmarks on [Real Roaring Datasets](https://github.com/RoaringBitmap/real-roaring-datasets) +run the following: + +```sh +go get github.com/RoaringBitmap/real-roaring-datasets +BENCH_REAL_DATA=1 go test -bench BenchmarkRealData -run - +``` + +### Iterative use + +You can use roaring with gore: + +- go get -u github.com/motemen/gore +- Make sure that ``$GOPATH/bin`` is in your ``$PATH``. +- go get github/RoaringBitmap/roaring + +```go +$ gore +gore version 0.2.6 :help for help +gore> :import github.com/RoaringBitmap/roaring +gore> x:=roaring.New() +gore> x.Add(1) +gore> x.String() +"{1}" +``` + + +### Fuzzy testing + +You can help us test further the library with fuzzy testing: + + go get github.com/dvyukov/go-fuzz/go-fuzz + go get github.com/dvyukov/go-fuzz/go-fuzz-build + go test -tags=gofuzz -run=TestGenerateSmatCorpus + go-fuzz-build github.com/RoaringBitmap/roaring + go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 + +Let it run, and if the # of crashers is > 0, check out the reports in +the workdir where you should be able to find the panic goroutine stack +traces. + +### Alternative in Go + +There is a Go version wrapping the C/C++ implementation https://github.com/RoaringBitmap/gocroaring + +For an alternative implementation in Go, see https://github.com/fzandona/goroar +The two versions were written independently. + + +### Mailing list/discussion group + +https://groups.google.com/forum/#!forum/roaring-bitmaps diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go new file mode 100644 index 00000000..c3958682 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer.go @@ -0,0 +1,960 @@ +package roaring + +import ( + "fmt" +) + +//go:generate msgp -unexported + +type arrayContainer struct { + content []uint16 +} + +func (ac *arrayContainer) String() string { + s := "{" + for it := ac.getShortIterator(); it.hasNext(); { + s += fmt.Sprintf("%v, ", it.next()) + } + return s + "}" +} + +func (ac *arrayContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { + for k := 0; k < len(ac.content); k++ { + x[k+i] = uint32(ac.content[k]) | mask + } +} + +func (ac *arrayContainer) getShortIterator() shortIterable { + return &shortIterator{ac.content, 0} +} + +func (ac *arrayContainer) getManyIterator() manyIterable { + return &manyIterator{ac.content, 0} +} + +func (ac *arrayContainer) minimum() uint16 { + return ac.content[0] // assume not empty +} + +func (ac *arrayContainer) maximum() uint16 { + return ac.content[len(ac.content)-1] // assume not empty +} + +func (ac *arrayContainer) getSizeInBytes() int { + return ac.getCardinality() * 2 +} + +func (ac *arrayContainer) serializedSizeInBytes() int { + return ac.getCardinality() * 2 +} + +func arrayContainerSizeInBytes(card int) int { + return card * 2 +} + +// add the values in the range [firstOfRange,endx) +func (ac *arrayContainer) iaddRange(firstOfRange, endx int) container { + if firstOfRange >= endx { + return ac + } + indexstart := binarySearch(ac.content, uint16(firstOfRange)) + if indexstart < 0 { + indexstart = -indexstart - 1 + } + indexend := binarySearch(ac.content, uint16(endx-1)) + if indexend < 0 { + indexend = -indexend - 1 + } else { + indexend++ + } + rangelength := endx - firstOfRange + newcardinality := indexstart + (ac.getCardinality() - indexend) + rangelength + if newcardinality > arrayDefaultMaxSize { + a := ac.toBitmapContainer() + return a.iaddRange(firstOfRange, endx) + } + if cap(ac.content) < newcardinality { + tmp := make([]uint16, newcardinality, newcardinality) + copy(tmp[:indexstart], ac.content[:indexstart]) + copy(tmp[indexstart+rangelength:], ac.content[indexend:]) + + ac.content = tmp + } else { + ac.content = ac.content[:newcardinality] + copy(ac.content[indexstart+rangelength:], ac.content[indexend:]) + + } + for k := 0; k < rangelength; k++ { + ac.content[k+indexstart] = uint16(firstOfRange + k) + } + return ac +} + +// remove the values in the range [firstOfRange,endx) +func (ac *arrayContainer) iremoveRange(firstOfRange, endx int) container { + if firstOfRange >= endx { + return ac + } + indexstart := binarySearch(ac.content, uint16(firstOfRange)) + if indexstart < 0 { + indexstart = -indexstart - 1 + } + indexend := binarySearch(ac.content, uint16(endx-1)) + if indexend < 0 { + indexend = -indexend - 1 + } else { + indexend++ + } + rangelength := indexend - indexstart + answer := ac + copy(answer.content[indexstart:], ac.content[indexstart+rangelength:]) + answer.content = answer.content[:ac.getCardinality()-rangelength] + return answer +} + +// flip the values in the range [firstOfRange,endx) +func (ac *arrayContainer) not(firstOfRange, endx int) container { + if firstOfRange >= endx { + //p("arrayContainer.not(): exiting early with ac.clone()") + return ac.clone() + } + return ac.notClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1] +} + +// flip the values in the range [firstOfRange,lastOfRange] +func (ac *arrayContainer) notClose(firstOfRange, lastOfRange int) container { + if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange] + //p("arrayContainer.notClose(): exiting early with ac.clone()") + return ac.clone() + } + + // determine the span of array indices to be affected^M + startIndex := binarySearch(ac.content, uint16(firstOfRange)) + //p("startIndex=%v", startIndex) + if startIndex < 0 { + startIndex = -startIndex - 1 + } + lastIndex := binarySearch(ac.content, uint16(lastOfRange)) + //p("lastIndex=%v", lastIndex) + if lastIndex < 0 { + lastIndex = -lastIndex - 2 + } + currentValuesInRange := lastIndex - startIndex + 1 + spanToBeFlipped := lastOfRange - firstOfRange + 1 + newValuesInRange := spanToBeFlipped - currentValuesInRange + cardinalityChange := newValuesInRange - currentValuesInRange + newCardinality := len(ac.content) + cardinalityChange + //p("new card is %v", newCardinality) + if newCardinality > arrayDefaultMaxSize { + //p("new card over arrayDefaultMaxSize, so returning bitmap") + return ac.toBitmapContainer().not(firstOfRange, lastOfRange+1) + } + answer := newArrayContainer() + answer.content = make([]uint16, newCardinality, newCardinality) //a hack for sure + + copy(answer.content, ac.content[:startIndex]) + outPos := startIndex + inPos := startIndex + valInRange := firstOfRange + for ; valInRange <= lastOfRange && inPos <= lastIndex; valInRange++ { + if uint16(valInRange) != ac.content[inPos] { + answer.content[outPos] = uint16(valInRange) + outPos++ + } else { + inPos++ + } + } + + for ; valInRange <= lastOfRange; valInRange++ { + answer.content[outPos] = uint16(valInRange) + outPos++ + } + + for i := lastIndex + 1; i < len(ac.content); i++ { + answer.content[outPos] = ac.content[i] + outPos++ + } + answer.content = answer.content[:newCardinality] + return answer + +} + +func (ac *arrayContainer) equals(o container) bool { + + srb, ok := o.(*arrayContainer) + if ok { + // Check if the containers are the same object. + if ac == srb { + return true + } + + if len(srb.content) != len(ac.content) { + return false + } + + for i, v := range ac.content { + if v != srb.content[i] { + return false + } + } + return true + } + + // use generic comparison + bCard := o.getCardinality() + aCard := ac.getCardinality() + if bCard != aCard { + return false + } + + ait := ac.getShortIterator() + bit := o.getShortIterator() + for ait.hasNext() { + if bit.next() != ait.next() { + return false + } + } + return true +} + +func (ac *arrayContainer) toBitmapContainer() *bitmapContainer { + bc := newBitmapContainer() + bc.loadData(ac) + return bc + +} +func (ac *arrayContainer) iadd(x uint16) (wasNew bool) { + // Special case adding to the end of the container. + l := len(ac.content) + if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x { + ac.content = append(ac.content, x) + return true + } + + loc := binarySearch(ac.content, x) + + if loc < 0 { + s := ac.content + i := -loc - 1 + s = append(s, 0) + copy(s[i+1:], s[i:]) + s[i] = x + ac.content = s + return true + } + return false +} + +func (ac *arrayContainer) iaddReturnMinimized(x uint16) container { + // Special case adding to the end of the container. + l := len(ac.content) + if l > 0 && l < arrayDefaultMaxSize && ac.content[l-1] < x { + ac.content = append(ac.content, x) + return ac + } + + loc := binarySearch(ac.content, x) + + if loc < 0 { + if len(ac.content) >= arrayDefaultMaxSize { + a := ac.toBitmapContainer() + a.iadd(x) + return a + } + s := ac.content + i := -loc - 1 + s = append(s, 0) + copy(s[i+1:], s[i:]) + s[i] = x + ac.content = s + } + return ac +} + +// iremoveReturnMinimized is allowed to change the return type to minimize storage. +func (ac *arrayContainer) iremoveReturnMinimized(x uint16) container { + ac.iremove(x) + return ac +} + +func (ac *arrayContainer) iremove(x uint16) bool { + loc := binarySearch(ac.content, x) + if loc >= 0 { + s := ac.content + s = append(s[:loc], s[loc+1:]...) + ac.content = s + return true + } + return false +} + +func (ac *arrayContainer) remove(x uint16) container { + out := &arrayContainer{make([]uint16, len(ac.content))} + copy(out.content, ac.content[:]) + + loc := binarySearch(out.content, x) + if loc >= 0 { + s := out.content + s = append(s[:loc], s[loc+1:]...) + out.content = s + } + return out +} + +func (ac *arrayContainer) or(a container) container { + switch x := a.(type) { + case *arrayContainer: + return ac.orArray(x) + case *bitmapContainer: + return x.orArray(ac) + case *runContainer16: + if x.isFull() { + return x.clone() + } + return x.orArray(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) orCardinality(a container) int { + switch x := a.(type) { + case *arrayContainer: + return ac.orArrayCardinality(x) + case *bitmapContainer: + return x.orArrayCardinality(ac) + case *runContainer16: + return x.orArrayCardinality(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) ior(a container) container { + switch x := a.(type) { + case *arrayContainer: + return ac.iorArray(x) + case *bitmapContainer: + return a.(*bitmapContainer).orArray(ac) + //return ac.iorBitmap(x) // note: this does not make sense + case *runContainer16: + if x.isFull() { + return x.clone() + } + return ac.iorRun16(x) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) iorArray(value2 *arrayContainer) container { + value1 := ac + len1 := value1.getCardinality() + len2 := value2.getCardinality() + maxPossibleCardinality := len1 + len2 + if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap! + bc := newBitmapContainer() + for k := 0; k < len(value2.content); k++ { + v := value2.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + for k := 0; k < len(ac.content); k++ { + v := ac.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + bc.cardinality = int(popcntSlice(bc.bitmap)) + if bc.cardinality <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc + } + if maxPossibleCardinality > cap(value1.content) { + newcontent := make([]uint16, 0, maxPossibleCardinality) + copy(newcontent[len2:maxPossibleCardinality], ac.content[0:len1]) + ac.content = newcontent + } else { + copy(ac.content[len2:maxPossibleCardinality], ac.content[0:len1]) + } + nl := union2by2(value1.content[len2:maxPossibleCardinality], value2.content, ac.content) + ac.content = ac.content[:nl] // reslice to match actual used capacity + return ac +} + +// Note: such code does not make practical sense, except for lazy evaluations +func (ac *arrayContainer) iorBitmap(bc2 *bitmapContainer) container { + bc1 := ac.toBitmapContainer() + bc1.iorBitmap(bc2) + *ac = *newArrayContainerFromBitmap(bc1) + return ac +} + +func (ac *arrayContainer) iorRun16(rc *runContainer16) container { + bc1 := ac.toBitmapContainer() + bc2 := rc.toBitmapContainer() + bc1.iorBitmap(bc2) + *ac = *newArrayContainerFromBitmap(bc1) + return ac +} + +func (ac *arrayContainer) lazyIOR(a container) container { + switch x := a.(type) { + case *arrayContainer: + return ac.lazyIorArray(x) + case *bitmapContainer: + return ac.lazyIorBitmap(x) + case *runContainer16: + if x.isFull() { + return x.clone() + } + return ac.lazyIorRun16(x) + + } + panic("unsupported container type") +} + +func (ac *arrayContainer) lazyIorArray(ac2 *arrayContainer) container { + // TODO actually make this lazy + return ac.iorArray(ac2) +} + +func (ac *arrayContainer) lazyIorBitmap(bc *bitmapContainer) container { + // TODO actually make this lazy + return ac.iorBitmap(bc) +} + +func (ac *arrayContainer) lazyIorRun16(rc *runContainer16) container { + // TODO actually make this lazy + return ac.iorRun16(rc) +} + +func (ac *arrayContainer) lazyOR(a container) container { + switch x := a.(type) { + case *arrayContainer: + return ac.lazyorArray(x) + case *bitmapContainer: + return a.lazyOR(ac) + case *runContainer16: + if x.isFull() { + return x.clone() + } + return x.orArray(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) orArray(value2 *arrayContainer) container { + value1 := ac + maxPossibleCardinality := value1.getCardinality() + value2.getCardinality() + if maxPossibleCardinality > arrayDefaultMaxSize { // it could be a bitmap! + bc := newBitmapContainer() + for k := 0; k < len(value2.content); k++ { + v := value2.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + for k := 0; k < len(ac.content); k++ { + v := ac.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + bc.cardinality = int(popcntSlice(bc.bitmap)) + if bc.cardinality <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc + } + answer := newArrayContainerCapacity(maxPossibleCardinality) + nl := union2by2(value1.content, value2.content, answer.content) + answer.content = answer.content[:nl] // reslice to match actual used capacity + return answer +} + +func (ac *arrayContainer) orArrayCardinality(value2 *arrayContainer) int { + return union2by2Cardinality(ac.content, value2.content) +} + +func (ac *arrayContainer) lazyorArray(value2 *arrayContainer) container { + value1 := ac + maxPossibleCardinality := value1.getCardinality() + value2.getCardinality() + if maxPossibleCardinality > arrayLazyLowerBound { // it could be a bitmap!^M + bc := newBitmapContainer() + for k := 0; k < len(value2.content); k++ { + v := value2.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + for k := 0; k < len(ac.content); k++ { + v := ac.content[k] + i := uint(v) >> 6 + mask := uint64(1) << (v % 64) + bc.bitmap[i] |= mask + } + bc.cardinality = invalidCardinality + return bc + } + answer := newArrayContainerCapacity(maxPossibleCardinality) + nl := union2by2(value1.content, value2.content, answer.content) + answer.content = answer.content[:nl] // reslice to match actual used capacity + return answer +} + +func (ac *arrayContainer) and(a container) container { + //p("ac.and() called") + switch x := a.(type) { + case *arrayContainer: + return ac.andArray(x) + case *bitmapContainer: + return x.and(ac) + case *runContainer16: + if x.isFull() { + return ac.clone() + } + return x.andArray(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) andCardinality(a container) int { + switch x := a.(type) { + case *arrayContainer: + return ac.andArrayCardinality(x) + case *bitmapContainer: + return x.andCardinality(ac) + case *runContainer16: + return x.andArrayCardinality(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) intersects(a container) bool { + switch x := a.(type) { + case *arrayContainer: + return ac.intersectsArray(x) + case *bitmapContainer: + return x.intersects(ac) + case *runContainer16: + return x.intersects(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) iand(a container) container { + switch x := a.(type) { + case *arrayContainer: + return ac.iandArray(x) + case *bitmapContainer: + return ac.iandBitmap(x) + case *runContainer16: + if x.isFull() { + return ac.clone() + } + return x.andArray(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) iandBitmap(bc *bitmapContainer) container { + pos := 0 + c := ac.getCardinality() + for k := 0; k < c; k++ { + // branchless + v := ac.content[k] + ac.content[pos] = v + pos += int(bc.bitValue(v)) + } + ac.content = ac.content[:pos] + return ac + +} + +func (ac *arrayContainer) xor(a container) container { + switch x := a.(type) { + case *arrayContainer: + return ac.xorArray(x) + case *bitmapContainer: + return a.xor(ac) + case *runContainer16: + return x.xorArray(ac) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) xorArray(value2 *arrayContainer) container { + value1 := ac + totalCardinality := value1.getCardinality() + value2.getCardinality() + if totalCardinality > arrayDefaultMaxSize { // it could be a bitmap! + bc := newBitmapContainer() + for k := 0; k < len(value2.content); k++ { + v := value2.content[k] + i := uint(v) >> 6 + bc.bitmap[i] ^= (uint64(1) << (v % 64)) + } + for k := 0; k < len(ac.content); k++ { + v := ac.content[k] + i := uint(v) >> 6 + bc.bitmap[i] ^= (uint64(1) << (v % 64)) + } + bc.computeCardinality() + if bc.cardinality <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc + } + desiredCapacity := totalCardinality + answer := newArrayContainerCapacity(desiredCapacity) + length := exclusiveUnion2by2(value1.content, value2.content, answer.content) + answer.content = answer.content[:length] + return answer + +} + +func (ac *arrayContainer) andNot(a container) container { + switch x := a.(type) { + case *arrayContainer: + return ac.andNotArray(x) + case *bitmapContainer: + return ac.andNotBitmap(x) + case *runContainer16: + return ac.andNotRun16(x) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) andNotRun16(rc *runContainer16) container { + acb := ac.toBitmapContainer() + rcb := rc.toBitmapContainer() + return acb.andNotBitmap(rcb) +} + +func (ac *arrayContainer) iandNot(a container) container { + switch x := a.(type) { + case *arrayContainer: + return ac.iandNotArray(x) + case *bitmapContainer: + return ac.iandNotBitmap(x) + case *runContainer16: + return ac.iandNotRun16(x) + } + panic("unsupported container type") +} + +func (ac *arrayContainer) iandNotRun16(rc *runContainer16) container { + rcb := rc.toBitmapContainer() + acb := ac.toBitmapContainer() + acb.iandNotBitmapSurely(rcb) + *ac = *(acb.toArrayContainer()) + return ac +} + +func (ac *arrayContainer) andNotArray(value2 *arrayContainer) container { + value1 := ac + desiredcapacity := value1.getCardinality() + answer := newArrayContainerCapacity(desiredcapacity) + length := difference(value1.content, value2.content, answer.content) + answer.content = answer.content[:length] + return answer +} + +func (ac *arrayContainer) iandNotArray(value2 *arrayContainer) container { + length := difference(ac.content, value2.content, ac.content) + ac.content = ac.content[:length] + return ac +} + +func (ac *arrayContainer) andNotBitmap(value2 *bitmapContainer) container { + desiredcapacity := ac.getCardinality() + answer := newArrayContainerCapacity(desiredcapacity) + answer.content = answer.content[:desiredcapacity] + pos := 0 + for _, v := range ac.content { + answer.content[pos] = v + pos += 1 - int(value2.bitValue(v)) + } + answer.content = answer.content[:pos] + return answer +} + +func (ac *arrayContainer) andBitmap(value2 *bitmapContainer) container { + desiredcapacity := ac.getCardinality() + answer := newArrayContainerCapacity(desiredcapacity) + answer.content = answer.content[:desiredcapacity] + pos := 0 + for _, v := range ac.content { + answer.content[pos] = v + pos += int(value2.bitValue(v)) + } + answer.content = answer.content[:pos] + return answer +} + +func (ac *arrayContainer) iandNotBitmap(value2 *bitmapContainer) container { + pos := 0 + for _, v := range ac.content { + ac.content[pos] = v + pos += 1 - int(value2.bitValue(v)) + } + ac.content = ac.content[:pos] + return ac +} + +func copyOf(array []uint16, size int) []uint16 { + result := make([]uint16, size) + for i, x := range array { + if i == size { + break + } + result[i] = x + } + return result +} + +// flip the values in the range [firstOfRange,endx) +func (ac *arrayContainer) inot(firstOfRange, endx int) container { + if firstOfRange >= endx { + return ac + } + return ac.inotClose(firstOfRange, endx-1) // remove everything in [firstOfRange,endx-1] +} + +// flip the values in the range [firstOfRange,lastOfRange] +func (ac *arrayContainer) inotClose(firstOfRange, lastOfRange int) container { + //p("ac.inotClose() starting") + if firstOfRange > lastOfRange { // unlike add and remove, not uses an inclusive range [firstOfRange,lastOfRange] + return ac + } + // determine the span of array indices to be affected + startIndex := binarySearch(ac.content, uint16(firstOfRange)) + if startIndex < 0 { + startIndex = -startIndex - 1 + } + lastIndex := binarySearch(ac.content, uint16(lastOfRange)) + if lastIndex < 0 { + lastIndex = -lastIndex - 1 - 1 + } + currentValuesInRange := lastIndex - startIndex + 1 + spanToBeFlipped := lastOfRange - firstOfRange + 1 + + newValuesInRange := spanToBeFlipped - currentValuesInRange + buffer := make([]uint16, newValuesInRange) + cardinalityChange := newValuesInRange - currentValuesInRange + newCardinality := len(ac.content) + cardinalityChange + if cardinalityChange > 0 { + if newCardinality > len(ac.content) { + if newCardinality > arrayDefaultMaxSize { + //p("ac.inotClose() converting to bitmap and doing inot there") + bcRet := ac.toBitmapContainer() + bcRet.inot(firstOfRange, lastOfRange+1) + *ac = *bcRet.toArrayContainer() + return bcRet + } + ac.content = copyOf(ac.content, newCardinality) + } + base := lastIndex + 1 + copy(ac.content[lastIndex+1+cardinalityChange:], ac.content[base:base+len(ac.content)-1-lastIndex]) + ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1) + } else { // no expansion needed + ac.negateRange(buffer, startIndex, lastIndex, firstOfRange, lastOfRange+1) + if cardinalityChange < 0 { + + for i := startIndex + newValuesInRange; i < newCardinality; i++ { + ac.content[i] = ac.content[i-cardinalityChange] + } + } + } + ac.content = ac.content[:newCardinality] + //p("bottom of ac.inotClose(): returning ac") + return ac +} + +func (ac *arrayContainer) negateRange(buffer []uint16, startIndex, lastIndex, startRange, lastRange int) { + // compute the negation into buffer + outPos := 0 + inPos := startIndex // value here always >= valInRange, + // until it is exhausted + // n.b., we can start initially exhausted. + + valInRange := startRange + for ; valInRange < lastRange && inPos <= lastIndex; valInRange++ { + if uint16(valInRange) != ac.content[inPos] { + buffer[outPos] = uint16(valInRange) + outPos++ + } else { + inPos++ + } + } + + // if there are extra items (greater than the biggest + // pre-existing one in range), buffer them + for ; valInRange < lastRange; valInRange++ { + buffer[outPos] = uint16(valInRange) + outPos++ + } + + if outPos != len(buffer) { + panic("negateRange: internal bug") + } + + for i, item := range buffer { + ac.content[i+startIndex] = item + } +} + +func (ac *arrayContainer) isFull() bool { + return false +} + +func (ac *arrayContainer) andArray(value2 *arrayContainer) container { + desiredcapacity := minOfInt(ac.getCardinality(), value2.getCardinality()) + answer := newArrayContainerCapacity(desiredcapacity) + length := intersection2by2( + ac.content, + value2.content, + answer.content) + answer.content = answer.content[:length] + return answer +} + +func (ac *arrayContainer) andArrayCardinality(value2 *arrayContainer) int { + return intersection2by2Cardinality( + ac.content, + value2.content) +} + +func (ac *arrayContainer) intersectsArray(value2 *arrayContainer) bool { + return intersects2by2( + ac.content, + value2.content) +} + +func (ac *arrayContainer) iandArray(value2 *arrayContainer) container { + length := intersection2by2( + ac.content, + value2.content, + ac.content) + ac.content = ac.content[:length] + return ac +} + +func (ac *arrayContainer) getCardinality() int { + return len(ac.content) +} + +func (ac *arrayContainer) rank(x uint16) int { + answer := binarySearch(ac.content, x) + if answer >= 0 { + return answer + 1 + } + return -answer - 1 + +} + +func (ac *arrayContainer) selectInt(x uint16) int { + return int(ac.content[x]) +} + +func (ac *arrayContainer) clone() container { + ptr := arrayContainer{make([]uint16, len(ac.content))} + copy(ptr.content, ac.content[:]) + return &ptr +} + +func (ac *arrayContainer) contains(x uint16) bool { + return binarySearch(ac.content, x) >= 0 +} + +func (ac *arrayContainer) loadData(bitmapContainer *bitmapContainer) { + ac.content = make([]uint16, bitmapContainer.cardinality, bitmapContainer.cardinality) + bitmapContainer.fillArray(ac.content) +} +func newArrayContainer() *arrayContainer { + p := new(arrayContainer) + return p +} + +func newArrayContainerFromBitmap(bc *bitmapContainer) *arrayContainer { + ac := &arrayContainer{} + ac.loadData(bc) + return ac +} + +func newArrayContainerCapacity(size int) *arrayContainer { + p := new(arrayContainer) + p.content = make([]uint16, 0, size) + return p +} + +func newArrayContainerSize(size int) *arrayContainer { + p := new(arrayContainer) + p.content = make([]uint16, size, size) + return p +} + +func newArrayContainerRange(firstOfRun, lastOfRun int) *arrayContainer { + valuesInRange := lastOfRun - firstOfRun + 1 + this := newArrayContainerCapacity(valuesInRange) + for i := 0; i < valuesInRange; i++ { + this.content = append(this.content, uint16(firstOfRun+i)) + } + return this +} + +func (ac *arrayContainer) numberOfRuns() (nr int) { + n := len(ac.content) + var runlen uint16 + var cur, prev uint16 + + switch n { + case 0: + return 0 + case 1: + return 1 + default: + for i := 1; i < n; i++ { + prev = ac.content[i-1] + cur = ac.content[i] + + if cur == prev+1 { + runlen++ + } else { + if cur < prev { + panic("then fundamental arrayContainer assumption of sorted ac.content was broken") + } + if cur == prev { + panic("then fundamental arrayContainer assumption of deduplicated content was broken") + } else { + nr++ + runlen = 0 + } + } + } + nr++ + } + return +} + +// convert to run or array *if needed* +func (ac *arrayContainer) toEfficientContainer() container { + + numRuns := ac.numberOfRuns() + + sizeAsRunContainer := runContainer16SerializedSizeInBytes(numRuns) + sizeAsBitmapContainer := bitmapContainerSizeInBytes() + card := ac.getCardinality() + sizeAsArrayContainer := arrayContainerSizeInBytes(card) + + if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { + return newRunContainer16FromArray(ac) + } + if card <= arrayDefaultMaxSize { + return ac + } + return ac.toBitmapContainer() +} + +func (ac *arrayContainer) containerType() contype { + return arrayContype +} diff --git a/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go new file mode 100644 index 00000000..cba6e53e --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go @@ -0,0 +1,134 @@ +package roaring + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import "github.com/tinylib/msgp/msgp" + +// DecodeMsg implements msgp.Decodable +func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zbzg uint32 + zbzg, err = dc.ReadMapHeader() + if err != nil { + return + } + for zbzg > 0 { + zbzg-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "content": + var zbai uint32 + zbai, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.content) >= int(zbai) { + z.content = (z.content)[:zbai] + } else { + z.content = make([]uint16, zbai) + } + for zxvk := range z.content { + z.content[zxvk], err = dc.ReadUint16() + if err != nil { + return + } + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "content" + err = en.Append(0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.content))) + if err != nil { + return + } + for zxvk := range z.content { + err = en.WriteUint16(z.content[zxvk]) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "content" + o = append(o, 0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74) + o = msgp.AppendArrayHeader(o, uint32(len(z.content))) + for zxvk := range z.content { + o = msgp.AppendUint16(o, z.content[zxvk]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zcmr uint32 + zcmr, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zcmr > 0 { + zcmr-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "content": + var zajw uint32 + zajw, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.content) >= int(zajw) { + z.content = (z.content)[:zajw] + } else { + z.content = make([]uint16, zajw) + } + for zxvk := range z.content { + z.content[zxvk], bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *arrayContainer) Msgsize() (s int) { + s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size)) + return +} diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go new file mode 100644 index 00000000..c4eeae18 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer.go @@ -0,0 +1,982 @@ +package roaring + +import ( + "fmt" + "unsafe" +) + +//go:generate msgp -unexported + +type bitmapContainer struct { + cardinality int + bitmap []uint64 +} + +func (bc bitmapContainer) String() string { + var s string + for it := bc.getShortIterator(); it.hasNext(); { + s += fmt.Sprintf("%v, ", it.next()) + } + return s +} + +func newBitmapContainer() *bitmapContainer { + p := new(bitmapContainer) + size := (1 << 16) / 64 + p.bitmap = make([]uint64, size, size) + return p +} + +func newBitmapContainerwithRange(firstOfRun, lastOfRun int) *bitmapContainer { + bc := newBitmapContainer() + bc.cardinality = lastOfRun - firstOfRun + 1 + if bc.cardinality == maxCapacity { + fill(bc.bitmap, uint64(0xffffffffffffffff)) + } else { + firstWord := firstOfRun / 64 + lastWord := lastOfRun / 64 + zeroPrefixLength := uint64(firstOfRun & 63) + zeroSuffixLength := uint64(63 - (lastOfRun & 63)) + + fillRange(bc.bitmap, firstWord, lastWord+1, uint64(0xffffffffffffffff)) + bc.bitmap[firstWord] ^= ((uint64(1) << zeroPrefixLength) - 1) + blockOfOnes := (uint64(1) << zeroSuffixLength) - 1 + maskOnLeft := blockOfOnes << (uint64(64) - zeroSuffixLength) + bc.bitmap[lastWord] ^= maskOnLeft + } + return bc +} + +func (bc *bitmapContainer) minimum() uint16 { + for i := 0; i < len(bc.bitmap); i++ { + w := bc.bitmap[i] + if w != 0 { + r := countTrailingZeros(w) + return uint16(r + i*64) + } + } + return MaxUint16 +} + +// i should be non-zero +func clz(i uint64) int { + n := 1 + x := uint32(i >> 32) + if x == 0 { + n += 32 + x = uint32(i) + } + if x>>16 == 0 { + n += 16 + x = x << 16 + } + if x>>24 == 0 { + n += 8 + x = x << 8 + } + if x>>28 == 0 { + n += 4 + x = x << 4 + } + if x>>30 == 0 { + n += 2 + x = x << 2 + } + return n - int(x>>31) +} + +func (bc *bitmapContainer) maximum() uint16 { + for i := len(bc.bitmap); i > 0; i-- { + w := bc.bitmap[i-1] + if w != 0 { + r := clz(w) + return uint16((i-1)*64 + 63 - r) + } + } + return uint16(0) +} + +type bitmapContainerShortIterator struct { + ptr *bitmapContainer + i int +} + +func (bcsi *bitmapContainerShortIterator) next() uint16 { + j := bcsi.i + bcsi.i = bcsi.ptr.NextSetBit(bcsi.i + 1) + return uint16(j) +} +func (bcsi *bitmapContainerShortIterator) hasNext() bool { + return bcsi.i >= 0 +} + +func newBitmapContainerShortIterator(a *bitmapContainer) *bitmapContainerShortIterator { + return &bitmapContainerShortIterator{a, a.NextSetBit(0)} +} + +func (bc *bitmapContainer) getShortIterator() shortIterable { + return newBitmapContainerShortIterator(bc) +} + +type bitmapContainerManyIterator struct { + ptr *bitmapContainer + base int + bitset uint64 +} + +func (bcmi *bitmapContainerManyIterator) nextMany(hs uint32, buf []uint32) int { + n := 0 + base := bcmi.base + bitset := bcmi.bitset + + for n < len(buf) { + if bitset == 0 { + base += 1 + if base >= len(bcmi.ptr.bitmap) { + bcmi.base = base + bcmi.bitset = bitset + return n + } + bitset = bcmi.ptr.bitmap[base] + continue + } + t := bitset & -bitset + buf[n] = uint32(((base * 64) + int(popcount(t-1)))) | hs + n = n + 1 + bitset ^= t + } + + bcmi.base = base + bcmi.bitset = bitset + return n +} + +func newBitmapContainerManyIterator(a *bitmapContainer) *bitmapContainerManyIterator { + return &bitmapContainerManyIterator{a, -1, 0} +} + +func (bc *bitmapContainer) getManyIterator() manyIterable { + return newBitmapContainerManyIterator(bc) +} + +func (bc *bitmapContainer) getSizeInBytes() int { + return len(bc.bitmap) * 8 // + bcBaseBytes +} + +func (bc *bitmapContainer) serializedSizeInBytes() int { + //return bc.Msgsize()// NOO! This breaks GetSerializedSizeInBytes + return len(bc.bitmap) * 8 +} + +const bcBaseBytes = int(unsafe.Sizeof(bitmapContainer{})) + +// bitmapContainer doesn't depend on card, always fully allocated +func bitmapContainerSizeInBytes() int { + return bcBaseBytes + (1<<16)/8 +} + +func bitmapEquals(a, b []uint64) bool { + if len(a) != len(b) { + //p("bitmaps differ on length. len(a)=%v; len(b)=%v", len(a), len(b)) + return false + } + for i, v := range a { + if v != b[i] { + //p("bitmaps differ on element i=%v", i) + return false + } + } + //p("bitmapEquals returning true") + return true +} + +func (bc *bitmapContainer) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { + // TODO: should be written as optimized assembly + pos := i + base := mask + for k := 0; k < len(bc.bitmap); k++ { + bitset := bc.bitmap[k] + for bitset != 0 { + t := bitset & -bitset + x[pos] = base + uint32(popcount(t-1)) + pos++ + bitset ^= t + } + base += 64 + } +} + +func (bc *bitmapContainer) equals(o container) bool { + srb, ok := o.(*bitmapContainer) + if ok { + //p("bitmapContainers.equals: both are bitmapContainers") + if srb.cardinality != bc.cardinality { + //p("bitmapContainers.equals: card differs: %v vs %v", srb.cardinality, bc.cardinality) + return false + } + return bitmapEquals(bc.bitmap, srb.bitmap) + } + + // use generic comparison + if bc.getCardinality() != o.getCardinality() { + return false + } + ait := o.getShortIterator() + bit := bc.getShortIterator() + + for ait.hasNext() { + if bit.next() != ait.next() { + return false + } + } + return true +} + +func (bc *bitmapContainer) iaddReturnMinimized(i uint16) container { + bc.iadd(i) + if bc.isFull() { + return newRunContainer16Range(0, MaxUint16) + } + return bc +} + +func (bc *bitmapContainer) iadd(i uint16) bool { + x := int(i) + previous := bc.bitmap[x/64] + mask := uint64(1) << (uint(x) % 64) + newb := previous | mask + bc.bitmap[x/64] = newb + bc.cardinality += int((previous ^ newb) >> (uint(x) % 64)) + return newb != previous +} + +func (bc *bitmapContainer) iremoveReturnMinimized(i uint16) container { + if bc.iremove(i) { + if bc.cardinality == arrayDefaultMaxSize { + return bc.toArrayContainer() + } + } + return bc +} + +// iremove returns true if i was found. +func (bc *bitmapContainer) iremove(i uint16) bool { + /* branchless code + w := bc.bitmap[i>>6] + mask := uint64(1) << (i % 64) + neww := w &^ mask + bc.cardinality -= int((w ^ neww) >> (i % 64)) + bc.bitmap[i>>6] = neww */ + if bc.contains(i) { + bc.cardinality-- + bc.bitmap[i/64] &^= (uint64(1) << (i % 64)) + return true + } + return false +} + +func (bc *bitmapContainer) isFull() bool { + return bc.cardinality == int(MaxUint16)+1 +} + +func (bc *bitmapContainer) getCardinality() int { + return bc.cardinality +} + +func (bc *bitmapContainer) clone() container { + ptr := bitmapContainer{bc.cardinality, make([]uint64, len(bc.bitmap))} + copy(ptr.bitmap, bc.bitmap[:]) + return &ptr +} + +// add all values in range [firstOfRange,lastOfRange) +func (bc *bitmapContainer) iaddRange(firstOfRange, lastOfRange int) container { + bc.cardinality += setBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, lastOfRange) + return bc +} + +// remove all values in range [firstOfRange,lastOfRange) +func (bc *bitmapContainer) iremoveRange(firstOfRange, lastOfRange int) container { + bc.cardinality += resetBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, lastOfRange) + if bc.getCardinality() <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc +} + +// flip all values in range [firstOfRange,endx) +func (bc *bitmapContainer) inot(firstOfRange, endx int) container { + p("bc.inot() called with [%v, %v)", firstOfRange, endx) + if endx-firstOfRange == maxCapacity { + //p("endx-firstOfRange == maxCapacity") + flipBitmapRange(bc.bitmap, firstOfRange, endx) + bc.cardinality = maxCapacity - bc.cardinality + //p("bc.cardinality is now %v", bc.cardinality) + } else if endx-firstOfRange > maxCapacity/2 { + //p("endx-firstOfRange > maxCapacity/2") + flipBitmapRange(bc.bitmap, firstOfRange, endx) + bc.computeCardinality() + } else { + bc.cardinality += flipBitmapRangeAndCardinalityChange(bc.bitmap, firstOfRange, endx) + } + if bc.getCardinality() <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc +} + +// flip all values in range [firstOfRange,endx) +func (bc *bitmapContainer) not(firstOfRange, endx int) container { + answer := bc.clone() + return answer.inot(firstOfRange, endx) +} + +func (bc *bitmapContainer) or(a container) container { + switch x := a.(type) { + case *arrayContainer: + return bc.orArray(x) + case *bitmapContainer: + return bc.orBitmap(x) + case *runContainer16: + if x.isFull() { + return x.clone() + } + return x.orBitmapContainer(bc) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) orCardinality(a container) int { + switch x := a.(type) { + case *arrayContainer: + return bc.orArrayCardinality(x) + case *bitmapContainer: + return bc.orBitmapCardinality(x) + case *runContainer16: + return x.orBitmapContainerCardinality(bc) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) ior(a container) container { + switch x := a.(type) { + case *arrayContainer: + return bc.iorArray(x) + case *bitmapContainer: + return bc.iorBitmap(x) + case *runContainer16: + if x.isFull() { + return x.clone() + } + for i := range x.iv { + bc.iaddRange(int(x.iv[i].start), int(x.iv[i].last())+1) + } + if bc.isFull() { + return newRunContainer16Range(0, MaxUint16) + } + //bc.computeCardinality() + return bc + } + panic(fmt.Errorf("unsupported container type %T", a)) +} + +func (bc *bitmapContainer) lazyIOR(a container) container { + switch x := a.(type) { + case *arrayContainer: + return bc.lazyIORArray(x) + case *bitmapContainer: + return bc.lazyIORBitmap(x) + case *runContainer16: + if x.isFull() { + return x.clone() + } + + // Manually inlined setBitmapRange function + bitmap := bc.bitmap + for _, iv := range x.iv { + start := int(iv.start) + end := int(iv.last()) + 1 + if start >= end { + continue + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + bitmap[firstword] |= (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) + continue + } + bitmap[firstword] |= ^uint64(0) << uint(start%64) + for i := firstword + 1; i < endword; i++ { + bitmap[i] = ^uint64(0) + } + bitmap[endword] |= ^uint64(0) >> (uint(-end) % 64) + } + bc.cardinality = invalidCardinality + return bc + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) lazyOR(a container) container { + switch x := a.(type) { + case *arrayContainer: + return bc.lazyORArray(x) + case *bitmapContainer: + return bc.lazyORBitmap(x) + case *runContainer16: + if x.isFull() { + return x.clone() + } + // TODO: implement lazy OR + return x.orBitmapContainer(bc) + + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) orArray(value2 *arrayContainer) container { + answer := bc.clone().(*bitmapContainer) + c := value2.getCardinality() + for k := 0; k < c; k++ { + v := value2.content[k] + i := uint(v) >> 6 + bef := answer.bitmap[i] + aft := bef | (uint64(1) << (v % 64)) + answer.bitmap[i] = aft + answer.cardinality += int((bef - aft) >> 63) + } + return answer +} + +func (bc *bitmapContainer) orArrayCardinality(value2 *arrayContainer) int { + answer := 0 + c := value2.getCardinality() + for k := 0; k < c; k++ { + // branchless: + v := value2.content[k] + i := uint(v) >> 6 + bef := bc.bitmap[i] + aft := bef | (uint64(1) << (v % 64)) + answer += int((bef - aft) >> 63) + } + return answer +} + +func (bc *bitmapContainer) orBitmap(value2 *bitmapContainer) container { + answer := newBitmapContainer() + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] + } + answer.computeCardinality() + if answer.isFull() { + return newRunContainer16Range(0, MaxUint16) + } + return answer +} + +func (bc *bitmapContainer) orBitmapCardinality(value2 *bitmapContainer) int { + return int(popcntOrSlice(bc.bitmap, value2.bitmap)) +} + +func (bc *bitmapContainer) andBitmapCardinality(value2 *bitmapContainer) int { + return int(popcntAndSlice(bc.bitmap, value2.bitmap)) +} + +func (bc *bitmapContainer) computeCardinality() { + bc.cardinality = int(popcntSlice(bc.bitmap)) +} + +func (bc *bitmapContainer) iorArray(ac *arrayContainer) container { + for k := range ac.content { + vc := ac.content[k] + i := uint(vc) >> 6 + bef := bc.bitmap[i] + aft := bef | (uint64(1) << (vc % 64)) + bc.bitmap[i] = aft + bc.cardinality += int((bef - aft) >> 63) + } + if bc.isFull() { + return newRunContainer16Range(0, MaxUint16) + } + return bc +} + +func (bc *bitmapContainer) iorBitmap(value2 *bitmapContainer) container { + answer := bc + answer.cardinality = 0 + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] + } + answer.computeCardinality() + if bc.isFull() { + return newRunContainer16Range(0, MaxUint16) + } + return answer +} + +func (bc *bitmapContainer) lazyIORArray(value2 *arrayContainer) container { + answer := bc + c := value2.getCardinality() + for k := 0; k < c; k++ { + vc := value2.content[k] + i := uint(vc) >> 6 + answer.bitmap[i] = answer.bitmap[i] | (uint64(1) << (vc % 64)) + } + answer.cardinality = invalidCardinality + return answer +} + +func (bc *bitmapContainer) lazyORArray(value2 *arrayContainer) container { + answer := bc.clone().(*bitmapContainer) + return answer.lazyIORArray(value2) +} + +func (bc *bitmapContainer) lazyIORBitmap(value2 *bitmapContainer) container { + answer := bc + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] | value2.bitmap[k] + } + bc.cardinality = invalidCardinality + return answer +} + +func (bc *bitmapContainer) lazyORBitmap(value2 *bitmapContainer) container { + answer := bc.clone().(*bitmapContainer) + return answer.lazyIORBitmap(value2) +} + +func (bc *bitmapContainer) xor(a container) container { + switch x := a.(type) { + case *arrayContainer: + return bc.xorArray(x) + case *bitmapContainer: + return bc.xorBitmap(x) + case *runContainer16: + return x.xorBitmap(bc) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) xorArray(value2 *arrayContainer) container { + answer := bc.clone().(*bitmapContainer) + c := value2.getCardinality() + for k := 0; k < c; k++ { + vc := value2.content[k] + index := uint(vc) >> 6 + abi := answer.bitmap[index] + mask := uint64(1) << (vc % 64) + answer.cardinality += 1 - 2*int((abi&mask)>>(vc%64)) + answer.bitmap[index] = abi ^ mask + } + if answer.cardinality <= arrayDefaultMaxSize { + return answer.toArrayContainer() + } + return answer +} + +func (bc *bitmapContainer) rank(x uint16) int { + // TODO: rewrite in assembly + leftover := (uint(x) + 1) & 63 + if leftover == 0 { + return int(popcntSlice(bc.bitmap[:(uint(x)+1)/64])) + } + return int(popcntSlice(bc.bitmap[:(uint(x)+1)/64]) + popcount(bc.bitmap[(uint(x)+1)/64]<<(64-leftover))) +} + +func (bc *bitmapContainer) selectInt(x uint16) int { + remaining := x + for k := 0; k < len(bc.bitmap); k++ { + w := popcount(bc.bitmap[k]) + if uint16(w) > remaining { + return k*64 + selectBitPosition(bc.bitmap[k], int(remaining)) + } + remaining -= uint16(w) + } + return -1 +} + +func (bc *bitmapContainer) xorBitmap(value2 *bitmapContainer) container { + newCardinality := int(popcntXorSlice(bc.bitmap, value2.bitmap)) + + if newCardinality > arrayDefaultMaxSize { + answer := newBitmapContainer() + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] ^ value2.bitmap[k] + } + answer.cardinality = newCardinality + if answer.isFull() { + return newRunContainer16Range(0, MaxUint16) + } + return answer + } + ac := newArrayContainerSize(newCardinality) + fillArrayXOR(ac.content, bc.bitmap, value2.bitmap) + ac.content = ac.content[:newCardinality] + return ac +} + +func (bc *bitmapContainer) and(a container) container { + switch x := a.(type) { + case *arrayContainer: + return bc.andArray(x) + case *bitmapContainer: + return bc.andBitmap(x) + case *runContainer16: + if x.isFull() { + return bc.clone() + } + return x.andBitmapContainer(bc) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) andCardinality(a container) int { + switch x := a.(type) { + case *arrayContainer: + return bc.andArrayCardinality(x) + case *bitmapContainer: + return bc.andBitmapCardinality(x) + case *runContainer16: + return x.andBitmapContainerCardinality(bc) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) intersects(a container) bool { + switch x := a.(type) { + case *arrayContainer: + return bc.intersectsArray(x) + case *bitmapContainer: + return bc.intersectsBitmap(x) + case *runContainer16: + return x.intersects(bc) + + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) iand(a container) container { + switch x := a.(type) { + case *arrayContainer: + return bc.iandArray(x) + case *bitmapContainer: + return bc.iandBitmap(x) + case *runContainer16: + if x.isFull() { + return bc.clone() + } + return bc.iandRun16(x) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) iandRun16(rc *runContainer16) container { + rcb := newBitmapContainerFromRun(rc) + return bc.iandBitmap(rcb) +} + +func (bc *bitmapContainer) iandArray(ac *arrayContainer) container { + acb := ac.toBitmapContainer() + return bc.iandBitmap(acb) +} + +func (bc *bitmapContainer) andArray(value2 *arrayContainer) *arrayContainer { + answer := newArrayContainerCapacity(len(value2.content)) + answer.content = answer.content[:cap(answer.content)] + c := value2.getCardinality() + pos := 0 + for k := 0; k < c; k++ { + v := value2.content[k] + answer.content[pos] = v + pos += int(bc.bitValue(v)) + } + answer.content = answer.content[:pos] + return answer +} + +func (bc *bitmapContainer) andArrayCardinality(value2 *arrayContainer) int { + c := value2.getCardinality() + pos := 0 + for k := 0; k < c; k++ { + v := value2.content[k] + pos += int(bc.bitValue(v)) + } + return pos +} + +func (bc *bitmapContainer) getCardinalityInRange(start, end uint) int { + if start >= end { + return 0 + } + firstword := start / 64 + endword := (end - 1) / 64 + const allones = ^uint64(0) + if firstword == endword { + return int(popcount(bc.bitmap[firstword] & ((allones << (start % 64)) & (allones >> (64 - (end % 64)))))) + } + answer := popcount(bc.bitmap[firstword] & (allones << (start % 64))) + answer += popcntSlice(bc.bitmap[firstword+1 : endword]) + answer += popcount(bc.bitmap[endword] & (allones >> (64 - (end % 64)))) + return int(answer) +} + +func (bc *bitmapContainer) andBitmap(value2 *bitmapContainer) container { + newcardinality := int(popcntAndSlice(bc.bitmap, value2.bitmap)) + if newcardinality > arrayDefaultMaxSize { + answer := newBitmapContainer() + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] & value2.bitmap[k] + } + answer.cardinality = newcardinality + return answer + } + ac := newArrayContainerSize(newcardinality) + fillArrayAND(ac.content, bc.bitmap, value2.bitmap) + ac.content = ac.content[:newcardinality] //not sure why i need this + return ac + +} + +func (bc *bitmapContainer) intersectsArray(value2 *arrayContainer) bool { + c := value2.getCardinality() + for k := 0; k < c; k++ { + v := value2.content[k] + if bc.contains(v) { + return true + } + } + return false +} + +func (bc *bitmapContainer) intersectsBitmap(value2 *bitmapContainer) bool { + for k := 0; k < len(bc.bitmap); k++ { + if (bc.bitmap[k] & value2.bitmap[k]) != 0 { + return true + } + } + return false + +} + +func (bc *bitmapContainer) iandBitmap(value2 *bitmapContainer) container { + newcardinality := int(popcntAndSlice(bc.bitmap, value2.bitmap)) + for k := 0; k < len(bc.bitmap); k++ { + bc.bitmap[k] = bc.bitmap[k] & value2.bitmap[k] + } + bc.cardinality = newcardinality + + if newcardinality <= arrayDefaultMaxSize { + return newArrayContainerFromBitmap(bc) + } + return bc +} + +func (bc *bitmapContainer) andNot(a container) container { + switch x := a.(type) { + case *arrayContainer: + return bc.andNotArray(x) + case *bitmapContainer: + return bc.andNotBitmap(x) + case *runContainer16: + return bc.andNotRun16(x) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) andNotRun16(rc *runContainer16) container { + rcb := rc.toBitmapContainer() + return bc.andNotBitmap(rcb) +} + +func (bc *bitmapContainer) iandNot(a container) container { + //p("bitmapContainer.iandNot() starting") + + switch x := a.(type) { + case *arrayContainer: + return bc.iandNotArray(x) + case *bitmapContainer: + return bc.iandNotBitmapSurely(x) + case *runContainer16: + return bc.iandNotRun16(x) + } + panic("unsupported container type") +} + +func (bc *bitmapContainer) iandNotArray(ac *arrayContainer) container { + acb := ac.toBitmapContainer() + return bc.iandNotBitmapSurely(acb) +} + +func (bc *bitmapContainer) iandNotRun16(rc *runContainer16) container { + rcb := rc.toBitmapContainer() + return bc.iandNotBitmapSurely(rcb) +} + +func (bc *bitmapContainer) andNotArray(value2 *arrayContainer) container { + answer := bc.clone().(*bitmapContainer) + c := value2.getCardinality() + for k := 0; k < c; k++ { + vc := value2.content[k] + i := uint(vc) >> 6 + oldv := answer.bitmap[i] + newv := oldv &^ (uint64(1) << (vc % 64)) + answer.bitmap[i] = newv + answer.cardinality -= int((oldv ^ newv) >> (vc % 64)) + } + if answer.cardinality <= arrayDefaultMaxSize { + return answer.toArrayContainer() + } + return answer +} + +func (bc *bitmapContainer) andNotBitmap(value2 *bitmapContainer) container { + newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap)) + if newCardinality > arrayDefaultMaxSize { + answer := newBitmapContainer() + for k := 0; k < len(answer.bitmap); k++ { + answer.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k] + } + answer.cardinality = newCardinality + return answer + } + ac := newArrayContainerSize(newCardinality) + fillArrayANDNOT(ac.content, bc.bitmap, value2.bitmap) + return ac +} + +func (bc *bitmapContainer) iandNotBitmapSurely(value2 *bitmapContainer) *bitmapContainer { + newCardinality := int(popcntMaskSlice(bc.bitmap, value2.bitmap)) + for k := 0; k < len(bc.bitmap); k++ { + bc.bitmap[k] = bc.bitmap[k] &^ value2.bitmap[k] + } + bc.cardinality = newCardinality + return bc +} + +func (bc *bitmapContainer) contains(i uint16) bool { //testbit + x := uint(i) + w := bc.bitmap[x>>6] + mask := uint64(1) << (x & 63) + return (w & mask) != 0 +} + +func (bc *bitmapContainer) bitValue(i uint16) uint64 { + x := uint(i) + w := bc.bitmap[x>>6] + return (w >> (x & 63)) & 1 +} + +func (bc *bitmapContainer) loadData(arrayContainer *arrayContainer) { + bc.cardinality = arrayContainer.getCardinality() + c := arrayContainer.getCardinality() + for k := 0; k < c; k++ { + x := arrayContainer.content[k] + i := int(x) / 64 + bc.bitmap[i] |= (uint64(1) << uint(x%64)) + } +} + +func (bc *bitmapContainer) toArrayContainer() *arrayContainer { + ac := &arrayContainer{} + ac.loadData(bc) + return ac +} + +func (bc *bitmapContainer) fillArray(container []uint16) { + //TODO: rewrite in assembly + pos := 0 + base := 0 + for k := 0; k < len(bc.bitmap); k++ { + bitset := bc.bitmap[k] + for bitset != 0 { + t := bitset & -bitset + container[pos] = uint16((base + int(popcount(t-1)))) + pos = pos + 1 + bitset ^= t + } + base += 64 + } +} + +func (bc *bitmapContainer) NextSetBit(i int) int { + x := i / 64 + if x >= len(bc.bitmap) { + return -1 + } + w := bc.bitmap[x] + w = w >> uint(i%64) + if w != 0 { + return i + countTrailingZeros(w) + } + x++ + for ; x < len(bc.bitmap); x++ { + if bc.bitmap[x] != 0 { + return (x * 64) + countTrailingZeros(bc.bitmap[x]) + } + } + return -1 +} + +// reference the java implementation +// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/BitmapContainer.java#L875-L892 +// +func (bc *bitmapContainer) numberOfRuns() int { + if bc.cardinality == 0 { + return 0 + } + + var numRuns uint64 + nextWord := bc.bitmap[0] + + for i := 0; i < len(bc.bitmap)-1; i++ { + word := nextWord + nextWord = bc.bitmap[i+1] + numRuns += popcount((^word)&(word<<1)) + ((word >> 63) &^ nextWord) + } + + word := nextWord + numRuns += popcount((^word) & (word << 1)) + if (word & 0x8000000000000000) != 0 { + numRuns++ + } + + return int(numRuns) +} + +// convert to run or array *if needed* +func (bc *bitmapContainer) toEfficientContainer() container { + + numRuns := bc.numberOfRuns() + + sizeAsRunContainer := runContainer16SerializedSizeInBytes(numRuns) + sizeAsBitmapContainer := bitmapContainerSizeInBytes() + card := bc.getCardinality() + sizeAsArrayContainer := arrayContainerSizeInBytes(card) + + if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { + return newRunContainer16FromBitmapContainer(bc) + } + if card <= arrayDefaultMaxSize { + return bc.toArrayContainer() + } + return bc +} + +func newBitmapContainerFromRun(rc *runContainer16) *bitmapContainer { + + if len(rc.iv) == 1 { + return newBitmapContainerwithRange(int(rc.iv[0].start), int(rc.iv[0].last())) + } + + bc := newBitmapContainer() + for i := range rc.iv { + setBitmapRange(bc.bitmap, int(rc.iv[i].start), int(rc.iv[i].last())+1) + bc.cardinality += int(rc.iv[i].last()) + 1 - int(rc.iv[i].start) + } + //bc.computeCardinality() + return bc +} + +func (bc *bitmapContainer) containerType() contype { + return bitmapContype +} diff --git a/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go new file mode 100644 index 00000000..f6c053e6 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/bitmapcontainer_gen.go @@ -0,0 +1,415 @@ +package roaring + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import "github.com/tinylib/msgp/msgp" + +// DecodeMsg implements msgp.Decodable +func (z *bitmapContainer) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zbzg uint32 + zbzg, err = dc.ReadMapHeader() + if err != nil { + return + } + for zbzg > 0 { + zbzg-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "cardinality": + z.cardinality, err = dc.ReadInt() + if err != nil { + return + } + case "bitmap": + var zbai uint32 + zbai, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.bitmap) >= int(zbai) { + z.bitmap = (z.bitmap)[:zbai] + } else { + z.bitmap = make([]uint64, zbai) + } + for zxvk := range z.bitmap { + z.bitmap[zxvk], err = dc.ReadUint64() + if err != nil { + return + } + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *bitmapContainer) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "cardinality" + err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79) + if err != nil { + return err + } + err = en.WriteInt(z.cardinality) + if err != nil { + return + } + // write "bitmap" + err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.bitmap))) + if err != nil { + return + } + for zxvk := range z.bitmap { + err = en.WriteUint64(z.bitmap[zxvk]) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *bitmapContainer) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "cardinality" + o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79) + o = msgp.AppendInt(o, z.cardinality) + // string "bitmap" + o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70) + o = msgp.AppendArrayHeader(o, uint32(len(z.bitmap))) + for zxvk := range z.bitmap { + o = msgp.AppendUint64(o, z.bitmap[zxvk]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *bitmapContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zcmr uint32 + zcmr, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zcmr > 0 { + zcmr-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "cardinality": + z.cardinality, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + return + } + case "bitmap": + var zajw uint32 + zajw, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.bitmap) >= int(zajw) { + z.bitmap = (z.bitmap)[:zajw] + } else { + z.bitmap = make([]uint64, zajw) + } + for zxvk := range z.bitmap { + z.bitmap[zxvk], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *bitmapContainer) Msgsize() (s int) { + s = 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.bitmap) * (msgp.Uint64Size)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *bitmapContainerShortIterator) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zhct uint32 + zhct, err = dc.ReadMapHeader() + if err != nil { + return + } + for zhct > 0 { + zhct-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "ptr": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + return + } + z.ptr = nil + } else { + if z.ptr == nil { + z.ptr = new(bitmapContainer) + } + var zcua uint32 + zcua, err = dc.ReadMapHeader() + if err != nil { + return + } + for zcua > 0 { + zcua-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "cardinality": + z.ptr.cardinality, err = dc.ReadInt() + if err != nil { + return + } + case "bitmap": + var zxhx uint32 + zxhx, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.ptr.bitmap) >= int(zxhx) { + z.ptr.bitmap = (z.ptr.bitmap)[:zxhx] + } else { + z.ptr.bitmap = make([]uint64, zxhx) + } + for zwht := range z.ptr.bitmap { + z.ptr.bitmap[zwht], err = dc.ReadUint64() + if err != nil { + return + } + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + case "i": + z.i, err = dc.ReadInt() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *bitmapContainerShortIterator) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "ptr" + err = en.Append(0x82, 0xa3, 0x70, 0x74, 0x72) + if err != nil { + return err + } + if z.ptr == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + // map header, size 2 + // write "cardinality" + err = en.Append(0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79) + if err != nil { + return err + } + err = en.WriteInt(z.ptr.cardinality) + if err != nil { + return + } + // write "bitmap" + err = en.Append(0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.ptr.bitmap))) + if err != nil { + return + } + for zwht := range z.ptr.bitmap { + err = en.WriteUint64(z.ptr.bitmap[zwht]) + if err != nil { + return + } + } + } + // write "i" + err = en.Append(0xa1, 0x69) + if err != nil { + return err + } + err = en.WriteInt(z.i) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *bitmapContainerShortIterator) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "ptr" + o = append(o, 0x82, 0xa3, 0x70, 0x74, 0x72) + if z.ptr == nil { + o = msgp.AppendNil(o) + } else { + // map header, size 2 + // string "cardinality" + o = append(o, 0x82, 0xab, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79) + o = msgp.AppendInt(o, z.ptr.cardinality) + // string "bitmap" + o = append(o, 0xa6, 0x62, 0x69, 0x74, 0x6d, 0x61, 0x70) + o = msgp.AppendArrayHeader(o, uint32(len(z.ptr.bitmap))) + for zwht := range z.ptr.bitmap { + o = msgp.AppendUint64(o, z.ptr.bitmap[zwht]) + } + } + // string "i" + o = append(o, 0xa1, 0x69) + o = msgp.AppendInt(o, z.i) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *bitmapContainerShortIterator) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zlqf uint32 + zlqf, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zlqf > 0 { + zlqf-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "ptr": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.ptr = nil + } else { + if z.ptr == nil { + z.ptr = new(bitmapContainer) + } + var zdaf uint32 + zdaf, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zdaf > 0 { + zdaf-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "cardinality": + z.ptr.cardinality, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + return + } + case "bitmap": + var zpks uint32 + zpks, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.ptr.bitmap) >= int(zpks) { + z.ptr.bitmap = (z.ptr.bitmap)[:zpks] + } else { + z.ptr.bitmap = make([]uint64, zpks) + } + for zwht := range z.ptr.bitmap { + z.ptr.bitmap[zwht], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + case "i": + z.i, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *bitmapContainerShortIterator) Msgsize() (s int) { + s = 1 + 4 + if z.ptr == nil { + s += msgp.NilSize + } else { + s += 1 + 12 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.ptr.bitmap) * (msgp.Uint64Size)) + } + s += 2 + msgp.IntSize + return +} diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz.go b/vendor/github.com/RoaringBitmap/roaring/ctz.go new file mode 100644 index 00000000..e399ddde --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/ctz.go @@ -0,0 +1,11 @@ +// +build go1.9 +// "go1.9", from Go version 1.9 onward +// See https://golang.org/pkg/go/build/#hdr-Build_Constraints + +package roaring + +import "math/bits" + +func countTrailingZeros(x uint64) int { + return bits.TrailingZeros64(x) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go new file mode 100644 index 00000000..80220e6b --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/ctz_compat.go @@ -0,0 +1,71 @@ +// +build !go1.9 + +package roaring + +// Reuse of portions of go/src/math/big standard lib code +// under this license: +/* +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +const deBruijn32 = 0x077CB531 + +var deBruijn32Lookup = []byte{ + 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9, +} + +const deBruijn64 = 0x03f79d71b4ca8b09 + +var deBruijn64Lookup = []byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +// trailingZeroBits returns the number of consecutive least significant zero +// bits of x. +func countTrailingZeros(x uint64) int { + // x & -x leaves only the right-most bit set in the word. Let k be the + // index of that bit. Since only a single bit is set, the value is two + // to the power of k. Multiplying by a power of two is equivalent to + // left shifting, in this case by k bits. The de Bruijn constant is + // such that all six bit, consecutive substrings are distinct. + // Therefore, if we have a left shifted version of this constant we can + // find by how many bits it was shifted by looking at which six bit + // substring ended up at the top of the word. + // (Knuth, volume 4, section 7.3.1) + if x == 0 { + // We have to special case 0; the fomula + // below doesn't work for 0. + return 64 + } + return int(deBruijn64Lookup[((x&-x)*(deBruijn64))>>58]) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go new file mode 100644 index 00000000..762e500e --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/fastaggregation.go @@ -0,0 +1,215 @@ +package roaring + +import ( + "container/heap" +) + +// Or function that requires repairAfterLazy +func lazyOR(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for (pos1 < length1) && (pos2 < length2) { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) + pos1++ + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) + pos2++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + c1 := x1.highlowcontainer.getContainerAtIndex(pos1) + switch t := c1.(type) { + case *arrayContainer: + c1 = t.toBitmapContainer() + case *runContainer16: + if !t.isFull() { + c1 = t.toBitmapContainer() + } + } + + answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + if pos1 == length1 { + answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } else if pos2 == length2 { + answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// In-place Or function that requires repairAfterLazy +func (x1 *Bitmap) lazyOR(x2 *Bitmap) *Bitmap { + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for (pos1 < length1) && (pos2 < length2) { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + pos1++ + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + x1.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone()) + pos2++ + pos1++ + length1++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + c1 := x1.highlowcontainer.getContainerAtIndex(pos1) + switch t := c1.(type) { + case *arrayContainer: + c1 = t.toBitmapContainer() + case *runContainer16: + if !t.isFull() { + c1 = t.toBitmapContainer() + } + case *bitmapContainer: + c1 = x1.highlowcontainer.getWritableContainerAtIndex(pos1) + } + + x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2)) + x1.highlowcontainer.needCopyOnWrite[pos1] = false + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + if pos1 == length1 { + x1.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } + return x1 +} + +// to be called after lazy aggregates +func (x1 *Bitmap) repairAfterLazy() { + for pos := 0; pos < x1.highlowcontainer.size(); pos++ { + c := x1.highlowcontainer.getContainerAtIndex(pos) + switch c.(type) { + case *bitmapContainer: + if c.(*bitmapContainer).cardinality == invalidCardinality { + c = x1.highlowcontainer.getWritableContainerAtIndex(pos) + c.(*bitmapContainer).computeCardinality() + if c.(*bitmapContainer).getCardinality() <= arrayDefaultMaxSize { + x1.highlowcontainer.setContainerAtIndex(pos, c.(*bitmapContainer).toArrayContainer()) + } else if c.(*bitmapContainer).isFull() { + x1.highlowcontainer.setContainerAtIndex(pos, newRunContainer16Range(0, MaxUint16)) + } + } + } + } +} + +// FastAnd computes the intersection between many bitmaps quickly +// Compared to the And function, it can take many bitmaps as input, thus saving the trouble +// of manually calling "And" many times. +func FastAnd(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return NewBitmap() + } else if len(bitmaps) == 1 { + return bitmaps[0].Clone() + } + answer := And(bitmaps[0], bitmaps[1]) + for _, bm := range bitmaps[2:] { + answer.And(bm) + } + return answer +} + +// FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly. +// It might also be faster than calling Or repeatedly. +func FastOr(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return NewBitmap() + } else if len(bitmaps) == 1 { + return bitmaps[0].Clone() + } + answer := lazyOR(bitmaps[0], bitmaps[1]) + for _, bm := range bitmaps[2:] { + answer = answer.lazyOR(bm) + } + // here is where repairAfterLazy is called. + answer.repairAfterLazy() + return answer +} + +// HeapOr computes the union between many bitmaps quickly using a heap. +// It might be faster than calling Or repeatedly. +func HeapOr(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return NewBitmap() + } + // TODO: for better speed, we could do the operation lazily, see Java implementation + pq := make(priorityQueue, len(bitmaps)) + for i, bm := range bitmaps { + pq[i] = &item{bm, i} + } + heap.Init(&pq) + + for pq.Len() > 1 { + x1 := heap.Pop(&pq).(*item) + x2 := heap.Pop(&pq).(*item) + heap.Push(&pq, &item{Or(x1.value, x2.value), 0}) + } + return heap.Pop(&pq).(*item).value +} + +// HeapXor computes the symmetric difference between many bitmaps quickly (as opposed to calling Xor repeated). +// Internally, this function uses a heap. +// It might be faster than calling Xor repeatedly. +func HeapXor(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return NewBitmap() + } + + pq := make(priorityQueue, len(bitmaps)) + for i, bm := range bitmaps { + pq[i] = &item{bm, i} + } + heap.Init(&pq) + + for pq.Len() > 1 { + x1 := heap.Pop(&pq).(*item) + x2 := heap.Pop(&pq).(*item) + heap.Push(&pq, &item{Xor(x1.value, x2.value), 0}) + } + return heap.Pop(&pq).(*item).value +} diff --git a/vendor/github.com/RoaringBitmap/roaring/manyiterator.go b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go new file mode 100644 index 00000000..b4f630a7 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/manyiterator.go @@ -0,0 +1,23 @@ +package roaring + +type manyIterable interface { + nextMany(hs uint32, buf []uint32) int +} + +type manyIterator struct { + slice []uint16 + loc int +} + +func (si *manyIterator) nextMany(hs uint32, buf []uint32) int { + n := 0 + l := si.loc + s := si.slice + for n < len(buf) && l < len(s) { + buf[n] = uint32(s[l]) | hs + l++ + n++ + } + si.loc = l + return n +} diff --git a/vendor/github.com/RoaringBitmap/roaring/parallel.go b/vendor/github.com/RoaringBitmap/roaring/parallel.go new file mode 100644 index 00000000..09f94fe8 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/parallel.go @@ -0,0 +1,613 @@ +package roaring + +import ( + "container/heap" + "fmt" + "runtime" + "sync" +) + +var defaultWorkerCount = runtime.NumCPU() + +type bitmapContainerKey struct { + key uint16 + idx int + bitmap *Bitmap +} + +type multipleContainers struct { + key uint16 + containers []container + idx int +} + +type keyedContainer struct { + key uint16 + container container + idx int +} + +type bitmapContainerHeap []bitmapContainerKey + +func (h bitmapContainerHeap) Len() int { return len(h) } +func (h bitmapContainerHeap) Less(i, j int) bool { return h[i].key < h[j].key } +func (h bitmapContainerHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *bitmapContainerHeap) Push(x interface{}) { + // Push and Pop use pointer receivers because they modify the slice's length, + // not just its contents. + *h = append(*h, x.(bitmapContainerKey)) +} + +func (h *bitmapContainerHeap) Pop() interface{} { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + return x +} + +func (h bitmapContainerHeap) Peek() bitmapContainerKey { + return h[0] +} + +func (h *bitmapContainerHeap) popIncrementing() (key uint16, container container) { + k := h.Peek() + key = k.key + container = k.bitmap.highlowcontainer.containers[k.idx] + + newIdx := k.idx + 1 + if newIdx < k.bitmap.highlowcontainer.size() { + k = bitmapContainerKey{ + k.bitmap.highlowcontainer.keys[newIdx], + newIdx, + k.bitmap, + } + (*h)[0] = k + heap.Fix(h, 0) + } else { + heap.Pop(h) + } + + return +} + +func (h *bitmapContainerHeap) Next(containers []container) multipleContainers { + if h.Len() == 0 { + return multipleContainers{} + } + + key, container := h.popIncrementing() + containers = append(containers, container) + + for h.Len() > 0 && key == h.Peek().key { + _, container = h.popIncrementing() + containers = append(containers, container) + } + + return multipleContainers{ + key, + containers, + -1, + } +} + +func newBitmapContainerHeap(bitmaps ...*Bitmap) bitmapContainerHeap { + // Initialize heap + var h bitmapContainerHeap = make([]bitmapContainerKey, 0, len(bitmaps)) + for _, bitmap := range bitmaps { + if !bitmap.IsEmpty() { + key := bitmapContainerKey{ + bitmap.highlowcontainer.keys[0], + 0, + bitmap, + } + h = append(h, key) + } + } + + heap.Init(&h) + + return h +} + +func repairAfterLazy(c container) container { + switch t := c.(type) { + case *bitmapContainer: + if t.cardinality == invalidCardinality { + t.computeCardinality() + } + + if t.getCardinality() <= arrayDefaultMaxSize { + return t.toArrayContainer() + } else if c.(*bitmapContainer).isFull() { + return newRunContainer16Range(0, MaxUint16) + } + } + + return c +} + +func toBitmapContainer(c container) container { + switch t := c.(type) { + case *arrayContainer: + return t.toBitmapContainer() + case *runContainer16: + if !t.isFull() { + return t.toBitmapContainer() + } + } + return c +} + +func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) { + expectedKeys := -1 + appendedKeys := 0 + keys := make([]uint16, 0) + containers := make([]container, 0) + for appendedKeys != expectedKeys { + select { + case item := <-resultChan: + if len(keys) <= item.idx { + keys = append(keys, make([]uint16, item.idx-len(keys)+1)...) + containers = append(containers, make([]container, item.idx-len(containers)+1)...) + } + keys[item.idx] = item.key + containers[item.idx] = item.container + + appendedKeys++ + case msg := <-expectedKeysChan: + expectedKeys = msg + } + } + answer := &Bitmap{ + roaringArray{ + make([]uint16, 0, expectedKeys), + make([]container, 0, expectedKeys), + make([]bool, 0, expectedKeys), + false, + nil, + }, + } + for i := range keys { + if containers[i] != nil { // in case a resulting container was empty, see ParAnd function + answer.highlowcontainer.appendContainer(keys[i], containers[i], false) + } + } + + bitmapChan <- answer +} + +// ParHeapOr computes the union (OR) of all provided bitmaps in parallel, +// where the parameter "parallelism" determines how many workers are to be used +// (if it is set to 0, a default number of workers is chosen) +// ParHeapOr uses a heap to compute the union. For rare cases it might be faster than ParOr +func ParHeapOr(parallelism int, bitmaps ...*Bitmap) *Bitmap { + + bitmapCount := len(bitmaps) + if bitmapCount == 0 { + return NewBitmap() + } else if bitmapCount == 1 { + return bitmaps[0].Clone() + } + + if parallelism == 0 { + parallelism = defaultWorkerCount + } + + h := newBitmapContainerHeap(bitmaps...) + + bitmapChan := make(chan *Bitmap) + inputChan := make(chan multipleContainers, 128) + resultChan := make(chan keyedContainer, 32) + expectedKeysChan := make(chan int) + + pool := sync.Pool{ + New: func() interface{} { + return make([]container, 0, len(bitmaps)) + }, + } + + orFunc := func() { + // Assumes only structs with >=2 containers are passed + for input := range inputChan { + c := toBitmapContainer(input.containers[0]).lazyOR(input.containers[1]) + for _, next := range input.containers[2:] { + c = c.lazyIOR(next) + } + c = repairAfterLazy(c) + kx := keyedContainer{ + input.key, + c, + input.idx, + } + resultChan <- kx + pool.Put(input.containers[:0]) + } + } + + go appenderRoutine(bitmapChan, resultChan, expectedKeysChan) + + for i := 0; i < parallelism; i++ { + go orFunc() + } + + idx := 0 + for h.Len() > 0 { + ck := h.Next(pool.Get().([]container)) + if len(ck.containers) == 1 { + resultChan <- keyedContainer{ + ck.key, + ck.containers[0], + idx, + } + pool.Put(ck.containers[:0]) + } else { + ck.idx = idx + inputChan <- ck + } + idx++ + } + expectedKeysChan <- idx + + bitmap := <-bitmapChan + + close(inputChan) + close(resultChan) + close(expectedKeysChan) + + return bitmap +} + +// ParAnd computes the intersection (AND) of all provided bitmaps in parallel, +// where the parameter "parallelism" determines how many workers are to be used +// (if it is set to 0, a default number of workers is chosen) +func ParAnd(parallelism int, bitmaps ...*Bitmap) *Bitmap { + bitmapCount := len(bitmaps) + if bitmapCount == 0 { + return NewBitmap() + } else if bitmapCount == 1 { + return bitmaps[0].Clone() + } + + if parallelism == 0 { + parallelism = defaultWorkerCount + } + + h := newBitmapContainerHeap(bitmaps...) + + bitmapChan := make(chan *Bitmap) + inputChan := make(chan multipleContainers, 128) + resultChan := make(chan keyedContainer, 32) + expectedKeysChan := make(chan int) + + andFunc := func() { + // Assumes only structs with >=2 containers are passed + for input := range inputChan { + c := input.containers[0].and(input.containers[1]) + for _, next := range input.containers[2:] { + if c.getCardinality() == 0 { + break + } + c = c.iand(next) + } + + // Send a nil explicitly if the result of the intersection is an empty container + if c.getCardinality() == 0 { + c = nil + } + + kx := keyedContainer{ + input.key, + c, + input.idx, + } + resultChan <- kx + } + } + + go appenderRoutine(bitmapChan, resultChan, expectedKeysChan) + + for i := 0; i < parallelism; i++ { + go andFunc() + } + + idx := 0 + for h.Len() > 0 { + ck := h.Next(make([]container, 0, 4)) + if len(ck.containers) == bitmapCount { + ck.idx = idx + inputChan <- ck + idx++ + } + } + expectedKeysChan <- idx + + bitmap := <-bitmapChan + + close(inputChan) + close(resultChan) + close(expectedKeysChan) + + return bitmap +} + +// ParOr computes the union (OR) of all provided bitmaps in parallel, +// where the parameter "parallelism" determines how many workers are to be used +// (if it is set to 0, a default number of workers is chosen) +func ParOr(parallelism int, bitmaps ...*Bitmap) *Bitmap { + var lKey uint16 = MaxUint16 + var hKey uint16 = 0 + + bitmapsFiltered := bitmaps[:0] + for _, b := range bitmaps { + if !b.IsEmpty() { + bitmapsFiltered = append(bitmapsFiltered, b) + } + } + bitmaps = bitmapsFiltered + + for _, b := range bitmaps { + lKey = minOfUint16(lKey, b.highlowcontainer.keys[0]) + hKey = maxOfUint16(hKey, b.highlowcontainer.keys[b.highlowcontainer.size()-1]) + } + + if lKey == MaxUint16 && hKey == 0 { + return New() + } else if len(bitmaps) == 1 { + return bitmaps[0] + } + + keyRange := hKey - lKey + 1 + if keyRange == 1 { + // revert to FastOr. Since the key range is 0 + // no container-level aggregation parallelism is achievable + return FastOr(bitmaps...) + } + + if parallelism == 0 { + parallelism = defaultWorkerCount + } + + var chunkSize int + var chunkCount int + if parallelism*4 > int(keyRange) { + chunkSize = 1 + chunkCount = int(keyRange) + } else { + chunkCount = parallelism * 4 + chunkSize = (int(keyRange) + chunkCount - 1) / chunkCount + } + + if chunkCount*chunkSize < int(keyRange) { + // it's fine to panic to indicate an implementation error + panic(fmt.Sprintf("invariant check failed: chunkCount * chunkSize < keyRange, %d * %d < %d", chunkCount, chunkSize, keyRange)) + } + + chunks := make([]*roaringArray, chunkCount) + + chunkSpecChan := make(chan parChunkSpec, minOfInt(maxOfInt(64, 2*parallelism), int(chunkCount))) + chunkChan := make(chan parChunk, minOfInt(32, int(chunkCount))) + + orFunc := func() { + for spec := range chunkSpecChan { + ra := lazyOrOnRange(&bitmaps[0].highlowcontainer, &bitmaps[1].highlowcontainer, spec.start, spec.end) + for _, b := range bitmaps[2:] { + ra = lazyIOrOnRange(ra, &b.highlowcontainer, spec.start, spec.end) + } + + for i, c := range ra.containers { + ra.containers[i] = repairAfterLazy(c) + } + + chunkChan <- parChunk{ra, spec.idx} + } + } + + for i := 0; i < parallelism; i++ { + go orFunc() + } + + go func() { + for i := 0; i < chunkCount; i++ { + spec := parChunkSpec{ + start: uint16(int(lKey) + i*chunkSize), + end: uint16(minOfInt(int(lKey)+(i+1)*chunkSize-1, int(hKey))), + idx: int(i), + } + chunkSpecChan <- spec + } + }() + + chunksRemaining := chunkCount + for chunk := range chunkChan { + chunks[chunk.idx] = chunk.ra + chunksRemaining-- + if chunksRemaining == 0 { + break + } + } + close(chunkChan) + close(chunkSpecChan) + + containerCount := 0 + for _, chunk := range chunks { + containerCount += chunk.size() + } + + result := Bitmap{ + roaringArray{ + containers: make([]container, containerCount), + keys: make([]uint16, containerCount), + needCopyOnWrite: make([]bool, containerCount), + }, + } + + resultOffset := 0 + for _, chunk := range chunks { + copy(result.highlowcontainer.containers[resultOffset:], chunk.containers) + copy(result.highlowcontainer.keys[resultOffset:], chunk.keys) + copy(result.highlowcontainer.needCopyOnWrite[resultOffset:], chunk.needCopyOnWrite) + resultOffset += chunk.size() + } + + return &result +} + +type parChunkSpec struct { + start uint16 + end uint16 + idx int +} + +type parChunk struct { + ra *roaringArray + idx int +} + +func (c parChunk) size() int { + return c.ra.size() +} + +func parNaiveStartAt(ra *roaringArray, start uint16, last uint16) int { + for idx, key := range ra.keys { + if key >= start && key <= last { + return idx + } else if key > last { + break + } + } + return ra.size() +} + +func lazyOrOnRange(ra1, ra2 *roaringArray, start, last uint16) *roaringArray { + answer := newRoaringArray() + length1 := ra1.size() + length2 := ra2.size() + + idx1 := parNaiveStartAt(ra1, start, last) + idx2 := parNaiveStartAt(ra2, start, last) + + var key1 uint16 + var key2 uint16 + if idx1 < length1 && idx2 < length2 { + key1 = ra1.getKeyAtIndex(idx1) + key2 = ra2.getKeyAtIndex(idx2) + + for key1 <= last && key2 <= last { + + if key1 < key2 { + answer.appendCopy(*ra1, idx1) + idx1++ + if idx1 == length1 { + break + } + key1 = ra1.getKeyAtIndex(idx1) + } else if key1 > key2 { + answer.appendCopy(*ra2, idx2) + idx2++ + if idx2 == length2 { + break + } + key2 = ra2.getKeyAtIndex(idx2) + } else { + c1 := ra1.getFastContainerAtIndex(idx1, false) + + answer.appendContainer(key1, c1.lazyOR(ra2.getContainerAtIndex(idx2)), false) + idx1++ + idx2++ + if idx1 == length1 || idx2 == length2 { + break + } + + key1 = ra1.getKeyAtIndex(idx1) + key2 = ra2.getKeyAtIndex(idx2) + } + } + } + + if idx2 < length2 { + key2 = ra2.getKeyAtIndex(idx2) + for key2 <= last { + answer.appendCopy(*ra2, idx2) + idx2++ + if idx2 == length2 { + break + } + key2 = ra2.getKeyAtIndex(idx2) + } + } + + if idx1 < length1 { + key1 = ra1.getKeyAtIndex(idx1) + for key1 <= last { + answer.appendCopy(*ra1, idx1) + idx1++ + if idx1 == length1 { + break + } + key1 = ra1.getKeyAtIndex(idx1) + } + } + return answer +} + +func lazyIOrOnRange(ra1, ra2 *roaringArray, start, last uint16) *roaringArray { + length1 := ra1.size() + length2 := ra2.size() + + idx1 := 0 + idx2 := parNaiveStartAt(ra2, start, last) + + var key1 uint16 + var key2 uint16 + if idx1 < length1 && idx2 < length2 { + key1 = ra1.getKeyAtIndex(idx1) + key2 = ra2.getKeyAtIndex(idx2) + + for key1 <= last && key2 <= last { + if key1 < key2 { + idx1++ + if idx1 >= length1 { + break + } + key1 = ra1.getKeyAtIndex(idx1) + } else if key1 > key2 { + ra1.insertNewKeyValueAt(idx1, key2, ra2.getContainerAtIndex(idx2)) + ra1.needCopyOnWrite[idx1] = true + idx2++ + idx1++ + length1++ + if idx2 >= length2 { + break + } + key2 = ra2.getKeyAtIndex(idx2) + } else { + c1 := ra1.getFastContainerAtIndex(idx1, true) + + ra1.containers[idx1] = c1.lazyIOR(ra2.getContainerAtIndex(idx2)) + ra1.needCopyOnWrite[idx1] = false + idx1++ + idx2++ + if idx1 >= length1 || idx2 >= length2 { + break + } + + key1 = ra1.getKeyAtIndex(idx1) + key2 = ra2.getKeyAtIndex(idx2) + } + } + } + if idx2 < length2 { + key2 = ra2.getKeyAtIndex(idx2) + for key2 <= last { + ra1.appendCopy(*ra2, idx2) + idx2++ + if idx2 >= length2 { + break + } + key2 = ra2.getKeyAtIndex(idx2) + } + } + return ra1 +} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt.go b/vendor/github.com/RoaringBitmap/roaring/popcnt.go new file mode 100644 index 00000000..9d99508c --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt.go @@ -0,0 +1,11 @@ +// +build go1.9 +// "go1.9", from Go version 1.9 onward +// See https://golang.org/pkg/go/build/#hdr-Build_Constraints + +package roaring + +import "math/bits" + +func popcount(x uint64) uint64 { + return uint64(bits.OnesCount64(x)) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s b/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s new file mode 100644 index 00000000..1f13fa2e --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s @@ -0,0 +1,103 @@ +// +build amd64,!appengine,!go1.9 + +TEXT ·hasAsm(SB),4,$0-1 +MOVQ $1, AX +CPUID +SHRQ $23, CX +ANDQ $1, CX +MOVB CX, ret+0(FP) +RET + +#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 + +TEXT ·popcntSliceAsm(SB),4,$0-32 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntSliceEnd +popcntSliceLoop: +BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX +ADDQ DX, AX +ADDQ $8, SI +LOOP popcntSliceLoop +popcntSliceEnd: +MOVQ AX, ret+24(FP) +RET + +TEXT ·popcntMaskSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntMaskSliceEnd +MOVQ m+24(FP), DI +popcntMaskSliceLoop: +MOVQ (DI), DX +NOTQ DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntMaskSliceLoop +popcntMaskSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntAndSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntAndSliceEnd +MOVQ m+24(FP), DI +popcntAndSliceLoop: +MOVQ (DI), DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntAndSliceLoop +popcntAndSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntOrSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntOrSliceEnd +MOVQ m+24(FP), DI +popcntOrSliceLoop: +MOVQ (DI), DX +ORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntOrSliceLoop +popcntOrSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntXorSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntXorSliceEnd +MOVQ m+24(FP), DI +popcntXorSliceLoop: +MOVQ (DI), DX +XORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntXorSliceLoop +popcntXorSliceEnd: +MOVQ AX, ret+48(FP) +RET diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go new file mode 100644 index 00000000..882d7f4e --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go @@ -0,0 +1,67 @@ +// +build amd64,!appengine,!go1.9 + +package roaring + +// *** the following functions are defined in popcnt_amd64.s + +//go:noescape + +func hasAsm() bool + +// useAsm is a flag used to select the GO or ASM implementation of the popcnt function +var useAsm = hasAsm() + +//go:noescape + +func popcntSliceAsm(s []uint64) uint64 + +//go:noescape + +func popcntMaskSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntAndSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntOrSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntXorSliceAsm(s, m []uint64) uint64 + +func popcntSlice(s []uint64) uint64 { + if useAsm { + return popcntSliceAsm(s) + } + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + if useAsm { + return popcntMaskSliceAsm(s, m) + } + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + if useAsm { + return popcntAndSliceAsm(s, m) + } + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + if useAsm { + return popcntOrSliceAsm(s, m) + } + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + if useAsm { + return popcntXorSliceAsm(s, m) + } + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go new file mode 100644 index 00000000..7ae82d4c --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go @@ -0,0 +1,17 @@ +// +build !go1.9 + +package roaring + +// bit population count, take from +// https://code.google.com/p/go/issues/detail?id=4988#c11 +// credit: https://code.google.com/u/arnehormann/ +// credit: https://play.golang.org/p/U7SogJ7psJ +// credit: http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel +func popcount(x uint64) uint64 { + x -= (x >> 1) & 0x5555555555555555 + x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 + x += x >> 4 + x &= 0x0f0f0f0f0f0f0f0f + x *= 0x0101010101010101 + return x >> 56 +} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go new file mode 100644 index 00000000..edf2083f --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go @@ -0,0 +1,23 @@ +// +build !amd64 appengine go1.9 + +package roaring + +func popcntSlice(s []uint64) uint64 { + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/popcnt_slices.go b/vendor/github.com/RoaringBitmap/roaring/popcnt_slices.go new file mode 100644 index 00000000..d27c5f38 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/popcnt_slices.go @@ -0,0 +1,41 @@ +package roaring + +func popcntSliceGo(s []uint64) uint64 { + cnt := uint64(0) + for _, x := range s { + cnt += popcount(x) + } + return cnt +} + +func popcntMaskSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] &^ m[i]) + } + return cnt +} + +func popcntAndSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] & m[i]) + } + return cnt +} + +func popcntOrSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] | m[i]) + } + return cnt +} + +func popcntXorSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] ^ m[i]) + } + return cnt +} diff --git a/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go b/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go new file mode 100644 index 00000000..9259a681 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/priorityqueue.go @@ -0,0 +1,101 @@ +package roaring + +import "container/heap" + +///////////// +// The priorityQueue is used to keep Bitmaps sorted. +//////////// + +type item struct { + value *Bitmap + index int +} + +type priorityQueue []*item + +func (pq priorityQueue) Len() int { return len(pq) } + +func (pq priorityQueue) Less(i, j int) bool { + return pq[i].value.GetSizeInBytes() < pq[j].value.GetSizeInBytes() +} + +func (pq priorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *priorityQueue) Push(x interface{}) { + n := len(*pq) + item := x.(*item) + item.index = n + *pq = append(*pq, item) +} + +func (pq *priorityQueue) Pop() interface{} { + old := *pq + n := len(old) + item := old[n-1] + item.index = -1 // for safety + *pq = old[0 : n-1] + return item +} + +func (pq *priorityQueue) update(item *item, value *Bitmap) { + item.value = value + heap.Fix(pq, item.index) +} + +///////////// +// The containerPriorityQueue is used to keep the containers of various Bitmaps sorted. +//////////// + +type containeritem struct { + value *Bitmap + keyindex int + index int +} + +type containerPriorityQueue []*containeritem + +func (pq containerPriorityQueue) Len() int { return len(pq) } + +func (pq containerPriorityQueue) Less(i, j int) bool { + k1 := pq[i].value.highlowcontainer.getKeyAtIndex(pq[i].keyindex) + k2 := pq[j].value.highlowcontainer.getKeyAtIndex(pq[j].keyindex) + if k1 != k2 { + return k1 < k2 + } + c1 := pq[i].value.highlowcontainer.getContainerAtIndex(pq[i].keyindex) + c2 := pq[j].value.highlowcontainer.getContainerAtIndex(pq[j].keyindex) + + return c1.getCardinality() > c2.getCardinality() +} + +func (pq containerPriorityQueue) Swap(i, j int) { + pq[i], pq[j] = pq[j], pq[i] + pq[i].index = i + pq[j].index = j +} + +func (pq *containerPriorityQueue) Push(x interface{}) { + n := len(*pq) + item := x.(*containeritem) + item.index = n + *pq = append(*pq, item) +} + +func (pq *containerPriorityQueue) Pop() interface{} { + old := *pq + n := len(old) + item := old[n-1] + item.index = -1 // for safety + *pq = old[0 : n-1] + return item +} + +//func (pq *containerPriorityQueue) update(item *containeritem, value *Bitmap, keyindex int) { +// item.value = value +// item.keyindex = keyindex +// heap.Fix(pq, item.index) +//} diff --git a/vendor/github.com/RoaringBitmap/roaring/rle.go b/vendor/github.com/RoaringBitmap/roaring/rle.go new file mode 100644 index 00000000..8f3d4edd --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/rle.go @@ -0,0 +1,1667 @@ +package roaring + +// +// Copyright (c) 2016 by the roaring authors. +// Licensed under the Apache License, Version 2.0. +// +// We derive a few lines of code from the sort.Search +// function in the golang standard library. That function +// is Copyright 2009 The Go Authors, and licensed +// under the following BSD-style license. +/* +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +import ( + "fmt" + "sort" + "unsafe" +) + +//go:generate msgp -unexported + +// runContainer32 does run-length encoding of sets of +// uint32 integers. +type runContainer32 struct { + iv []interval32 + card int64 + + // avoid allocation during search + myOpts searchOptions `msg:"-"` +} + +// interval32 is the internal to runContainer32 +// structure that maintains the individual [Start, last] +// closed intervals. +type interval32 struct { + start uint32 + last uint32 +} + +// runlen returns the count of integers in the interval. +func (iv interval32) runlen() int64 { + return 1 + int64(iv.last) - int64(iv.start) +} + +// String produces a human viewable string of the contents. +func (iv interval32) String() string { + return fmt.Sprintf("[%d, %d]", iv.start, iv.last) +} + +func ivalString32(iv []interval32) string { + var s string + var j int + var p interval32 + for j, p = range iv { + s += fmt.Sprintf("%v:[%d, %d], ", j, p.start, p.last) + } + return s +} + +// String produces a human viewable string of the contents. +func (rc *runContainer32) String() string { + if len(rc.iv) == 0 { + return "runContainer32{}" + } + is := ivalString32(rc.iv) + return `runContainer32{` + is + `}` +} + +// uint32Slice is a sort.Sort convenience method +type uint32Slice []uint32 + +// Len returns the length of p. +func (p uint32Slice) Len() int { return len(p) } + +// Less returns p[i] < p[j] +func (p uint32Slice) Less(i, j int) bool { return p[i] < p[j] } + +// Swap swaps elements i and j. +func (p uint32Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +//msgp:ignore addHelper + +// addHelper helps build a runContainer32. +type addHelper32 struct { + runstart uint32 + runlen uint32 + actuallyAdded uint32 + m []interval32 + rc *runContainer32 +} + +func (ah *addHelper32) storeIval(runstart, runlen uint32) { + mi := interval32{start: runstart, last: runstart + runlen} + ah.m = append(ah.m, mi) +} + +func (ah *addHelper32) add(cur, prev uint32, i int) { + if cur == prev+1 { + ah.runlen++ + ah.actuallyAdded++ + } else { + if cur < prev { + panic(fmt.Sprintf("newRunContainer32FromVals sees "+ + "unsorted vals; vals[%v]=cur=%v < prev=%v. Sort your vals"+ + " before calling us with alreadySorted == true.", i, cur, prev)) + } + if cur == prev { + // ignore duplicates + } else { + ah.actuallyAdded++ + ah.storeIval(ah.runstart, ah.runlen) + ah.runstart = cur + ah.runlen = 0 + } + } +} + +// newRunContainerRange makes a new container made of just the specified closed interval [rangestart,rangelast] +func newRunContainer32Range(rangestart uint32, rangelast uint32) *runContainer32 { + rc := &runContainer32{} + rc.iv = append(rc.iv, interval32{start: rangestart, last: rangelast}) + return rc +} + +// newRunContainer32FromVals makes a new container from vals. +// +// For efficiency, vals should be sorted in ascending order. +// Ideally vals should not contain duplicates, but we detect and +// ignore them. If vals is already sorted in ascending order, then +// pass alreadySorted = true. Otherwise, for !alreadySorted, +// we will sort vals before creating a runContainer32 of them. +// We sort the original vals, so this will change what the +// caller sees in vals as a side effect. +func newRunContainer32FromVals(alreadySorted bool, vals ...uint32) *runContainer32 { + // keep this in sync with newRunContainer32FromArray below + + rc := &runContainer32{} + ah := addHelper32{rc: rc} + + if !alreadySorted { + sort.Sort(uint32Slice(vals)) + } + n := len(vals) + var cur, prev uint32 + switch { + case n == 0: + // nothing more + case n == 1: + ah.m = append(ah.m, interval32{start: vals[0], last: vals[0]}) + ah.actuallyAdded++ + default: + ah.runstart = vals[0] + ah.actuallyAdded++ + for i := 1; i < n; i++ { + prev = vals[i-1] + cur = vals[i] + ah.add(cur, prev, i) + } + ah.storeIval(ah.runstart, ah.runlen) + } + rc.iv = ah.m + rc.card = int64(ah.actuallyAdded) + return rc +} + +// newRunContainer32FromBitmapContainer makes a new run container from bc, +// somewhat efficiently. For reference, see the Java +// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/RunContainer.java#L145-L192 +func newRunContainer32FromBitmapContainer(bc *bitmapContainer) *runContainer32 { + + rc := &runContainer32{} + nbrRuns := bc.numberOfRuns() + if nbrRuns == 0 { + return rc + } + rc.iv = make([]interval32, nbrRuns) + + longCtr := 0 // index of current long in bitmap + curWord := bc.bitmap[0] // its value + runCount := 0 + for { + // potentially multiword advance to first 1 bit + for curWord == 0 && longCtr < len(bc.bitmap)-1 { + longCtr++ + curWord = bc.bitmap[longCtr] + } + + if curWord == 0 { + // wrap up, no more runs + return rc + } + localRunStart := countTrailingZeros(curWord) + runStart := localRunStart + 64*longCtr + // stuff 1s into number's LSBs + curWordWith1s := curWord | (curWord - 1) + + // find the next 0, potentially in a later word + runEnd := 0 + for curWordWith1s == maxWord && longCtr < len(bc.bitmap)-1 { + longCtr++ + curWordWith1s = bc.bitmap[longCtr] + } + + if curWordWith1s == maxWord { + // a final unterminated run of 1s + runEnd = wordSizeInBits + longCtr*64 + rc.iv[runCount].start = uint32(runStart) + rc.iv[runCount].last = uint32(runEnd) - 1 + return rc + } + localRunEnd := countTrailingZeros(^curWordWith1s) + runEnd = localRunEnd + longCtr*64 + rc.iv[runCount].start = uint32(runStart) + rc.iv[runCount].last = uint32(runEnd) - 1 + runCount++ + // now, zero out everything right of runEnd. + curWord = curWordWith1s & (curWordWith1s + 1) + // We've lathered and rinsed, so repeat... + } + +} + +// +// newRunContainer32FromArray populates a new +// runContainer32 from the contents of arr. +// +func newRunContainer32FromArray(arr *arrayContainer) *runContainer32 { + // keep this in sync with newRunContainer32FromVals above + + rc := &runContainer32{} + ah := addHelper32{rc: rc} + + n := arr.getCardinality() + var cur, prev uint32 + switch { + case n == 0: + // nothing more + case n == 1: + ah.m = append(ah.m, interval32{start: uint32(arr.content[0]), last: uint32(arr.content[0])}) + ah.actuallyAdded++ + default: + ah.runstart = uint32(arr.content[0]) + ah.actuallyAdded++ + for i := 1; i < n; i++ { + prev = uint32(arr.content[i-1]) + cur = uint32(arr.content[i]) + ah.add(cur, prev, i) + } + ah.storeIval(ah.runstart, ah.runlen) + } + rc.iv = ah.m + rc.card = int64(ah.actuallyAdded) + return rc +} + +// set adds the integers in vals to the set. Vals +// must be sorted in increasing order; if not, you should set +// alreadySorted to false, and we will sort them in place for you. +// (Be aware of this side effect -- it will affect the callers +// view of vals). +// +// If you have a small number of additions to an already +// big runContainer32, calling Add() may be faster. +func (rc *runContainer32) set(alreadySorted bool, vals ...uint32) { + + rc2 := newRunContainer32FromVals(alreadySorted, vals...) + un := rc.union(rc2) + rc.iv = un.iv + rc.card = 0 +} + +// canMerge returns true if the intervals +// a and b either overlap or they are +// contiguous and so can be merged into +// a single interval. +func canMerge32(a, b interval32) bool { + if int64(a.last)+1 < int64(b.start) { + return false + } + return int64(b.last)+1 >= int64(a.start) +} + +// haveOverlap differs from canMerge in that +// it tells you if the intersection of a +// and b would contain an element (otherwise +// it would be the empty set, and we return +// false). +func haveOverlap32(a, b interval32) bool { + if int64(a.last)+1 <= int64(b.start) { + return false + } + return int64(b.last)+1 > int64(a.start) +} + +// mergeInterval32s joins a and b into a +// new interval, and panics if it cannot. +func mergeInterval32s(a, b interval32) (res interval32) { + if !canMerge32(a, b) { + panic(fmt.Sprintf("cannot merge %#v and %#v", a, b)) + } + if b.start < a.start { + res.start = b.start + } else { + res.start = a.start + } + if b.last > a.last { + res.last = b.last + } else { + res.last = a.last + } + return +} + +// intersectInterval32s returns the intersection +// of a and b. The isEmpty flag will be true if +// a and b were disjoint. +func intersectInterval32s(a, b interval32) (res interval32, isEmpty bool) { + if !haveOverlap32(a, b) { + isEmpty = true + return + } + if b.start > a.start { + res.start = b.start + } else { + res.start = a.start + } + if b.last < a.last { + res.last = b.last + } else { + res.last = a.last + } + return +} + +// union merges two runContainer32s, producing +// a new runContainer32 with the union of rc and b. +func (rc *runContainer32) union(b *runContainer32) *runContainer32 { + + // rc is also known as 'a' here, but golint insisted we + // call it rc for consistency with the rest of the methods. + + var m []interval32 + + alim := int64(len(rc.iv)) + blim := int64(len(b.iv)) + + var na int64 // next from a + var nb int64 // next from b + + // merged holds the current merge output, which might + // get additional merges before being appended to m. + var merged interval32 + var mergedUsed bool // is merged being used at the moment? + + var cura interval32 // currently considering this interval32 from a + var curb interval32 // currently considering this interval32 from b + + pass := 0 + for na < alim && nb < blim { + pass++ + cura = rc.iv[na] + curb = b.iv[nb] + + if mergedUsed { + mergedUpdated := false + if canMerge32(cura, merged) { + merged = mergeInterval32s(cura, merged) + na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) + mergedUpdated = true + } + if canMerge32(curb, merged) { + merged = mergeInterval32s(curb, merged) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) + mergedUpdated = true + } + if !mergedUpdated { + // we know that merged is disjoint from cura and curb + m = append(m, merged) + mergedUsed = false + } + continue + + } else { + // !mergedUsed + if !canMerge32(cura, curb) { + if cura.start < curb.start { + m = append(m, cura) + na++ + } else { + m = append(m, curb) + nb++ + } + } else { + merged = mergeInterval32s(cura, curb) + mergedUsed = true + na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) + } + } + } + var aDone, bDone bool + if na >= alim { + aDone = true + } + if nb >= blim { + bDone = true + } + // finish by merging anything remaining into merged we can: + if mergedUsed { + if !aDone { + aAdds: + for na < alim { + cura = rc.iv[na] + if canMerge32(cura, merged) { + merged = mergeInterval32s(cura, merged) + na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) + } else { + break aAdds + } + } + + } + + if !bDone { + bAdds: + for nb < blim { + curb = b.iv[nb] + if canMerge32(curb, merged) { + merged = mergeInterval32s(curb, merged) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) + } else { + break bAdds + } + } + + } + + m = append(m, merged) + } + if na < alim { + m = append(m, rc.iv[na:]...) + } + if nb < blim { + m = append(m, b.iv[nb:]...) + } + + res := &runContainer32{iv: m} + return res +} + +// unionCardinality returns the cardinality of the merger of two runContainer32s, the union of rc and b. +func (rc *runContainer32) unionCardinality(b *runContainer32) uint64 { + + // rc is also known as 'a' here, but golint insisted we + // call it rc for consistency with the rest of the methods. + answer := uint64(0) + + alim := int64(len(rc.iv)) + blim := int64(len(b.iv)) + + var na int64 // next from a + var nb int64 // next from b + + // merged holds the current merge output, which might + // get additional merges before being appended to m. + var merged interval32 + var mergedUsed bool // is merged being used at the moment? + + var cura interval32 // currently considering this interval32 from a + var curb interval32 // currently considering this interval32 from b + + pass := 0 + for na < alim && nb < blim { + pass++ + cura = rc.iv[na] + curb = b.iv[nb] + + if mergedUsed { + mergedUpdated := false + if canMerge32(cura, merged) { + merged = mergeInterval32s(cura, merged) + na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) + mergedUpdated = true + } + if canMerge32(curb, merged) { + merged = mergeInterval32s(curb, merged) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) + mergedUpdated = true + } + if !mergedUpdated { + // we know that merged is disjoint from cura and curb + //m = append(m, merged) + answer += uint64(merged.last) - uint64(merged.start) + 1 + mergedUsed = false + } + continue + + } else { + // !mergedUsed + if !canMerge32(cura, curb) { + if cura.start < curb.start { + answer += uint64(cura.last) - uint64(cura.start) + 1 + //m = append(m, cura) + na++ + } else { + answer += uint64(curb.last) - uint64(curb.start) + 1 + //m = append(m, curb) + nb++ + } + } else { + merged = mergeInterval32s(cura, curb) + mergedUsed = true + na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) + } + } + } + var aDone, bDone bool + if na >= alim { + aDone = true + } + if nb >= blim { + bDone = true + } + // finish by merging anything remaining into merged we can: + if mergedUsed { + if !aDone { + aAdds: + for na < alim { + cura = rc.iv[na] + if canMerge32(cura, merged) { + merged = mergeInterval32s(cura, merged) + na = rc.indexOfIntervalAtOrAfter(int64(merged.last)+1, na+1) + } else { + break aAdds + } + } + + } + + if !bDone { + bAdds: + for nb < blim { + curb = b.iv[nb] + if canMerge32(curb, merged) { + merged = mergeInterval32s(curb, merged) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last)+1, nb+1) + } else { + break bAdds + } + } + + } + + //m = append(m, merged) + answer += uint64(merged.last) - uint64(merged.start) + 1 + } + for _, r := range rc.iv[na:] { + answer += uint64(r.last) - uint64(r.start) + 1 + } + for _, r := range b.iv[nb:] { + answer += uint64(r.last) - uint64(r.start) + 1 + } + return answer +} + +// indexOfIntervalAtOrAfter is a helper for union. +func (rc *runContainer32) indexOfIntervalAtOrAfter(key int64, startIndex int64) int64 { + rc.myOpts.startIndex = startIndex + rc.myOpts.endxIndex = 0 + + w, already, _ := rc.search(key, &rc.myOpts) + if already { + return w + } + return w + 1 +} + +// intersect returns a new runContainer32 holding the +// intersection of rc (also known as 'a') and b. +func (rc *runContainer32) intersect(b *runContainer32) *runContainer32 { + + a := rc + numa := int64(len(a.iv)) + numb := int64(len(b.iv)) + res := &runContainer32{} + if numa == 0 || numb == 0 { + return res + } + + if numa == 1 && numb == 1 { + if !haveOverlap32(a.iv[0], b.iv[0]) { + return res + } + } + + var output []interval32 + + var acuri int64 + var bcuri int64 + + astart := int64(a.iv[acuri].start) + bstart := int64(b.iv[bcuri].start) + + var intersection interval32 + var leftoverstart int64 + var isOverlap, isLeftoverA, isLeftoverB bool + var done bool + pass := 0 +toploop: + for acuri < numa && bcuri < numb { + pass++ + + isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = intersectWithLeftover32(astart, int64(a.iv[acuri].last), bstart, int64(b.iv[bcuri].last)) + + if !isOverlap { + switch { + case astart < bstart: + acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart) + if done { + break toploop + } + astart = int64(a.iv[acuri].start) + + case astart > bstart: + bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) + if done { + break toploop + } + bstart = int64(b.iv[bcuri].start) + + //default: + // panic("impossible that astart == bstart, since !isOverlap") + } + + } else { + // isOverlap + output = append(output, intersection) + switch { + case isLeftoverA: + // note that we change astart without advancing acuri, + // since we need to capture any 2ndary intersections with a.iv[acuri] + astart = leftoverstart + bcuri++ + if bcuri >= numb { + break toploop + } + bstart = int64(b.iv[bcuri].start) + case isLeftoverB: + // note that we change bstart without advancing bcuri, + // since we need to capture any 2ndary intersections with b.iv[bcuri] + bstart = leftoverstart + acuri++ + if acuri >= numa { + break toploop + } + astart = int64(a.iv[acuri].start) + default: + // neither had leftover, both completely consumed + // optionally, assert for sanity: + //if a.iv[acuri].endx != b.iv[bcuri].endx { + // panic("huh? should only be possible that endx agree now!") + //} + + // advance to next a interval + acuri++ + if acuri >= numa { + break toploop + } + astart = int64(a.iv[acuri].start) + + // advance to next b interval + bcuri++ + if bcuri >= numb { + break toploop + } + bstart = int64(b.iv[bcuri].start) + } + } + } // end for toploop + + if len(output) == 0 { + return res + } + + res.iv = output + return res +} + +// intersectCardinality returns the cardinality of the +// intersection of rc (also known as 'a') and b. +func (rc *runContainer32) intersectCardinality(b *runContainer32) int64 { + answer := int64(0) + + a := rc + numa := int64(len(a.iv)) + numb := int64(len(b.iv)) + if numa == 0 || numb == 0 { + return 0 + } + + if numa == 1 && numb == 1 { + if !haveOverlap32(a.iv[0], b.iv[0]) { + return 0 + } + } + + var acuri int64 + var bcuri int64 + + astart := int64(a.iv[acuri].start) + bstart := int64(b.iv[bcuri].start) + + var intersection interval32 + var leftoverstart int64 + var isOverlap, isLeftoverA, isLeftoverB bool + var done bool + pass := 0 +toploop: + for acuri < numa && bcuri < numb { + pass++ + + isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = intersectWithLeftover32(astart, int64(a.iv[acuri].last), bstart, int64(b.iv[bcuri].last)) + + if !isOverlap { + switch { + case astart < bstart: + acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart) + if done { + break toploop + } + astart = int64(a.iv[acuri].start) + + case astart > bstart: + bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) + if done { + break toploop + } + bstart = int64(b.iv[bcuri].start) + + //default: + // panic("impossible that astart == bstart, since !isOverlap") + } + + } else { + // isOverlap + answer += int64(intersection.last) - int64(intersection.start) + 1 + switch { + case isLeftoverA: + // note that we change astart without advancing acuri, + // since we need to capture any 2ndary intersections with a.iv[acuri] + astart = leftoverstart + bcuri++ + if bcuri >= numb { + break toploop + } + bstart = int64(b.iv[bcuri].start) + case isLeftoverB: + // note that we change bstart without advancing bcuri, + // since we need to capture any 2ndary intersections with b.iv[bcuri] + bstart = leftoverstart + acuri++ + if acuri >= numa { + break toploop + } + astart = int64(a.iv[acuri].start) + default: + // neither had leftover, both completely consumed + // optionally, assert for sanity: + //if a.iv[acuri].endx != b.iv[bcuri].endx { + // panic("huh? should only be possible that endx agree now!") + //} + + // advance to next a interval + acuri++ + if acuri >= numa { + break toploop + } + astart = int64(a.iv[acuri].start) + + // advance to next b interval + bcuri++ + if bcuri >= numb { + break toploop + } + bstart = int64(b.iv[bcuri].start) + } + } + } // end for toploop + + return answer +} + +// get returns true if key is in the container. +func (rc *runContainer32) contains(key uint32) bool { + _, in, _ := rc.search(int64(key), nil) + return in +} + +// numIntervals returns the count of intervals in the container. +func (rc *runContainer32) numIntervals() int { + return len(rc.iv) +} + +// search returns alreadyPresent to indicate if the +// key is already in one of our interval32s. +// +// If key is alreadyPresent, then whichInterval32 tells +// you where. +// +// If key is not already present, then whichInterval32 is +// set as follows: +// +// a) whichInterval32 == len(rc.iv)-1 if key is beyond our +// last interval32 in rc.iv; +// +// b) whichInterval32 == -1 if key is before our first +// interval32 in rc.iv; +// +// c) whichInterval32 is set to the minimum index of rc.iv +// which comes strictly before the key; +// so rc.iv[whichInterval32].last < key, +// and if whichInterval32+1 exists, then key < rc.iv[whichInterval32+1].start +// (Note that whichInterval32+1 won't exist when +// whichInterval32 is the last interval.) +// +// runContainer32.search always returns whichInterval32 < len(rc.iv). +// +// If not nil, opts can be used to further restrict +// the search space. +// +func (rc *runContainer32) search(key int64, opts *searchOptions) (whichInterval32 int64, alreadyPresent bool, numCompares int) { + n := int64(len(rc.iv)) + if n == 0 { + return -1, false, 0 + } + + startIndex := int64(0) + endxIndex := n + if opts != nil { + startIndex = opts.startIndex + + // let endxIndex == 0 mean no effect + if opts.endxIndex > 0 { + endxIndex = opts.endxIndex + } + } + + // sort.Search returns the smallest index i + // in [0, n) at which f(i) is true, assuming that on the range [0, n), + // f(i) == true implies f(i+1) == true. + // If there is no such index, Search returns n. + + // For correctness, this began as verbatim snippet from + // sort.Search in the Go standard lib. + // We inline our comparison function for speed, and + // annotate with numCompares + // to observe and test that extra bounds are utilized. + i, j := startIndex, endxIndex + for i < j { + h := i + (j-i)/2 // avoid overflow when computing h as the bisector + // i <= h < j + numCompares++ + if !(key < int64(rc.iv[h].start)) { + i = h + 1 + } else { + j = h + } + } + below := i + // end std lib snippet. + + // The above is a simple in-lining and annotation of: + /* below := sort.Search(n, + func(i int) bool { + return key < rc.iv[i].start + }) + */ + whichInterval32 = below - 1 + + if below == n { + // all falses => key is >= start of all interval32s + // ... so does it belong to the last interval32? + if key < int64(rc.iv[n-1].last)+1 { + // yes, it belongs to the last interval32 + alreadyPresent = true + return + } + // no, it is beyond the last interval32. + // leave alreadyPreset = false + return + } + + // INVAR: key is below rc.iv[below] + if below == 0 { + // key is before the first first interval32. + // leave alreadyPresent = false + return + } + + // INVAR: key is >= rc.iv[below-1].start and + // key is < rc.iv[below].start + + // is key in below-1 interval32? + if key >= int64(rc.iv[below-1].start) && key < int64(rc.iv[below-1].last)+1 { + // yes, it is. key is in below-1 interval32. + alreadyPresent = true + return + } + + // INVAR: key >= rc.iv[below-1].endx && key < rc.iv[below].start + // leave alreadyPresent = false + return +} + +// cardinality returns the count of the integers stored in the +// runContainer32. +func (rc *runContainer32) cardinality() int64 { + if len(rc.iv) == 0 { + rc.card = 0 + return 0 + } + if rc.card > 0 { + return rc.card // already cached + } + // have to compute it + var n int64 + for _, p := range rc.iv { + n += p.runlen() + } + rc.card = n // cache it + return n +} + +// AsSlice decompresses the contents into a []uint32 slice. +func (rc *runContainer32) AsSlice() []uint32 { + s := make([]uint32, rc.cardinality()) + j := 0 + for _, p := range rc.iv { + for i := p.start; i <= p.last; i++ { + s[j] = i + j++ + } + } + return s +} + +// newRunContainer32 creates an empty run container. +func newRunContainer32() *runContainer32 { + return &runContainer32{} +} + +// newRunContainer32CopyIv creates a run container, initializing +// with a copy of the supplied iv slice. +// +func newRunContainer32CopyIv(iv []interval32) *runContainer32 { + rc := &runContainer32{ + iv: make([]interval32, len(iv)), + } + copy(rc.iv, iv) + return rc +} + +func (rc *runContainer32) Clone() *runContainer32 { + rc2 := newRunContainer32CopyIv(rc.iv) + return rc2 +} + +// newRunContainer32TakeOwnership returns a new runContainer32 +// backed by the provided iv slice, which we will +// assume exclusive control over from now on. +// +func newRunContainer32TakeOwnership(iv []interval32) *runContainer32 { + rc := &runContainer32{ + iv: iv, + } + return rc +} + +const baseRc32Size = int(unsafe.Sizeof(runContainer32{})) +const perIntervalRc32Size = int(unsafe.Sizeof(interval32{})) + +const baseDiskRc32Size = int(unsafe.Sizeof(uint32(0))) + +// see also runContainer32SerializedSizeInBytes(numRuns int) int + +// getSizeInBytes returns the number of bytes of memory +// required by this runContainer32. +func (rc *runContainer32) getSizeInBytes() int { + return perIntervalRc32Size*len(rc.iv) + baseRc32Size +} + +// runContainer32SerializedSizeInBytes returns the number of bytes of disk +// required to hold numRuns in a runContainer32. +func runContainer32SerializedSizeInBytes(numRuns int) int { + return perIntervalRc32Size*numRuns + baseDiskRc32Size +} + +// Add adds a single value k to the set. +func (rc *runContainer32) Add(k uint32) (wasNew bool) { + // TODO comment from runContainer32.java: + // it might be better and simpler to do return + // toBitmapOrArrayContainer(getCardinality()).add(k) + // but note that some unit tests use this method to build up test + // runcontainers without calling runOptimize + + k64 := int64(k) + + index, present, _ := rc.search(k64, nil) + if present { + return // already there + } + wasNew = true + + // increment card if it is cached already + if rc.card > 0 { + rc.card++ + } + n := int64(len(rc.iv)) + if index == -1 { + // we may need to extend the first run + if n > 0 { + if rc.iv[0].start == k+1 { + rc.iv[0].start = k + return + } + } + // nope, k stands alone, starting the new first interval32. + rc.iv = append([]interval32{{start: k, last: k}}, rc.iv...) + return + } + + // are we off the end? handle both index == n and index == n-1: + if index >= n-1 { + if int64(rc.iv[n-1].last)+1 == k64 { + rc.iv[n-1].last++ + return + } + rc.iv = append(rc.iv, interval32{start: k, last: k}) + return + } + + // INVAR: index and index+1 both exist, and k goes between them. + // + // Now: add k into the middle, + // possibly fusing with index or index+1 interval32 + // and possibly resulting in fusing of two interval32s + // that had a one integer gap. + + left := index + right := index + 1 + + // are we fusing left and right by adding k? + if int64(rc.iv[left].last)+1 == k64 && int64(rc.iv[right].start) == k64+1 { + // fuse into left + rc.iv[left].last = rc.iv[right].last + // remove redundant right + rc.iv = append(rc.iv[:left+1], rc.iv[right+1:]...) + return + } + + // are we an addition to left? + if int64(rc.iv[left].last)+1 == k64 { + // yes + rc.iv[left].last++ + return + } + + // are we an addition to right? + if int64(rc.iv[right].start) == k64+1 { + // yes + rc.iv[right].start = k + return + } + + // k makes a standalone new interval32, inserted in the middle + tail := append([]interval32{{start: k, last: k}}, rc.iv[right:]...) + rc.iv = append(rc.iv[:left+1], tail...) + return +} + +//msgp:ignore runIterator + +// runIterator32 advice: you must call Next() at least once +// before calling Cur(); and you should call HasNext() +// before calling Next() to insure there are contents. +type runIterator32 struct { + rc *runContainer32 + curIndex int64 + curPosInIndex uint32 + curSeq int64 +} + +// newRunIterator32 returns a new empty run container. +func (rc *runContainer32) newRunIterator32() *runIterator32 { + return &runIterator32{rc: rc, curIndex: -1} +} + +// HasNext returns false if calling Next will panic. It +// returns true when there is at least one more value +// available in the iteration sequence. +func (ri *runIterator32) hasNext() bool { + if len(ri.rc.iv) == 0 { + return false + } + if ri.curIndex == -1 { + return true + } + return ri.curSeq+1 < ri.rc.cardinality() +} + +// cur returns the current value pointed to by the iterator. +func (ri *runIterator32) cur() uint32 { + return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex +} + +// Next returns the next value in the iteration sequence. +func (ri *runIterator32) next() uint32 { + if !ri.hasNext() { + panic("no Next available") + } + if ri.curIndex >= int64(len(ri.rc.iv)) { + panic("runIterator.Next() going beyond what is available") + } + if ri.curIndex == -1 { + // first time is special + ri.curIndex = 0 + } else { + ri.curPosInIndex++ + if int64(ri.rc.iv[ri.curIndex].start)+int64(ri.curPosInIndex) == int64(ri.rc.iv[ri.curIndex].last)+1 { + ri.curPosInIndex = 0 + ri.curIndex++ + } + ri.curSeq++ + } + return ri.cur() +} + +// remove removes the element that the iterator +// is on from the run container. You can use +// Cur if you want to double check what is about +// to be deleted. +func (ri *runIterator32) remove() uint32 { + n := ri.rc.cardinality() + if n == 0 { + panic("runIterator.Remove called on empty runContainer32") + } + cur := ri.cur() + + ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq) + return cur +} + +// remove removes key from the container. +func (rc *runContainer32) removeKey(key uint32) (wasPresent bool) { + + var index int64 + var curSeq int64 + index, wasPresent, _ = rc.search(int64(key), nil) + if !wasPresent { + return // already removed, nothing to do. + } + pos := key - rc.iv[index].start + rc.deleteAt(&index, &pos, &curSeq) + return +} + +// internal helper functions + +func (rc *runContainer32) deleteAt(curIndex *int64, curPosInIndex *uint32, curSeq *int64) { + rc.card-- + (*curSeq)-- + ci := *curIndex + pos := *curPosInIndex + + // are we first, last, or in the middle of our interval32? + switch { + case pos == 0: + if int64(rc.iv[ci].start) == int64(rc.iv[ci].last) { + // our interval disappears + rc.iv = append(rc.iv[:ci], rc.iv[ci+1:]...) + // curIndex stays the same, since the delete did + // the advance for us. + *curPosInIndex = 0 + } else { + rc.iv[ci].start++ // no longer overflowable + } + case int64(pos) == rc.iv[ci].runlen()-1: + // last + rc.iv[ci].last-- + // our interval32 cannot disappear, else we would have been pos == 0, case first above. + (*curPosInIndex)-- + // if we leave *curIndex alone, then Next() will work properly even after the delete. + default: + //middle + // split into two, adding an interval32 + new0 := interval32{ + start: rc.iv[ci].start, + last: rc.iv[ci].start + *curPosInIndex - 1} + + new1start := int64(rc.iv[ci].start) + int64(*curPosInIndex) + 1 + if new1start > int64(MaxUint32) { + panic("overflow?!?!") + } + new1 := interval32{ + start: uint32(new1start), + last: rc.iv[ci].last} + tail := append([]interval32{new0, new1}, rc.iv[ci+1:]...) + rc.iv = append(rc.iv[:ci], tail...) + // update curIndex and curPosInIndex + (*curIndex)++ + *curPosInIndex = 0 + } + +} + +func have4Overlap32(astart, alast, bstart, blast int64) bool { + if alast+1 <= bstart { + return false + } + return blast+1 > astart +} + +func intersectWithLeftover32(astart, alast, bstart, blast int64) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int64, intersection interval32) { + if !have4Overlap32(astart, alast, bstart, blast) { + return + } + isOverlap = true + + // do the intersection: + if bstart > astart { + intersection.start = uint32(bstart) + } else { + intersection.start = uint32(astart) + } + switch { + case blast < alast: + isLeftoverA = true + leftoverstart = blast + 1 + intersection.last = uint32(blast) + case alast < blast: + isLeftoverB = true + leftoverstart = alast + 1 + intersection.last = uint32(alast) + default: + // alast == blast + intersection.last = uint32(alast) + } + + return +} + +func (rc *runContainer32) findNextIntervalThatIntersectsStartingFrom(startIndex int64, key int64) (index int64, done bool) { + + rc.myOpts.startIndex = startIndex + rc.myOpts.endxIndex = 0 + + w, _, _ := rc.search(key, &rc.myOpts) + // rc.search always returns w < len(rc.iv) + if w < startIndex { + // not found and comes before lower bound startIndex, + // so just use the lower bound. + if startIndex == int64(len(rc.iv)) { + // also this bump up means that we are done + return startIndex, true + } + return startIndex, false + } + + return w, false +} + +func sliceToString32(m []interval32) string { + s := "" + for i := range m { + s += fmt.Sprintf("%v: %s, ", i, m[i]) + } + return s +} + +// selectInt32 returns the j-th value in the container. +// We panic of j is out of bounds. +func (rc *runContainer32) selectInt32(j uint32) int { + n := rc.cardinality() + if int64(j) > n { + panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n)) + } + + var offset int64 + for k := range rc.iv { + nextOffset := offset + rc.iv[k].runlen() + 1 + if nextOffset > int64(j) { + return int(int64(rc.iv[k].start) + (int64(j) - offset)) + } + offset = nextOffset + } + panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n)) +} + +// helper for invert +func (rc *runContainer32) invertlastInterval(origin uint32, lastIdx int) []interval32 { + cur := rc.iv[lastIdx] + if cur.last == MaxUint32 { + if cur.start == origin { + return nil // empty container + } + return []interval32{{start: origin, last: cur.start - 1}} + } + if cur.start == origin { + return []interval32{{start: cur.last + 1, last: MaxUint32}} + } + // invert splits + return []interval32{ + {start: origin, last: cur.start - 1}, + {start: cur.last + 1, last: MaxUint32}, + } +} + +// invert returns a new container (not inplace), that is +// the inversion of rc. For each bit b in rc, the +// returned value has !b +func (rc *runContainer32) invert() *runContainer32 { + ni := len(rc.iv) + var m []interval32 + switch ni { + case 0: + return &runContainer32{iv: []interval32{{0, MaxUint32}}} + case 1: + return &runContainer32{iv: rc.invertlastInterval(0, 0)} + } + var invstart int64 + ult := ni - 1 + for i, cur := range rc.iv { + if i == ult { + // invertlastInteval will add both intervals (b) and (c) in + // diagram below. + m = append(m, rc.invertlastInterval(uint32(invstart), i)...) + break + } + // INVAR: i and cur are not the last interval, there is a next at i+1 + // + // ........[cur.start, cur.last] ...... [next.start, next.last].... + // ^ ^ ^ + // (a) (b) (c) + // + // Now: we add interval (a); but if (a) is empty, for cur.start==0, we skip it. + if cur.start > 0 { + m = append(m, interval32{start: uint32(invstart), last: cur.start - 1}) + } + invstart = int64(cur.last + 1) + } + return &runContainer32{iv: m} +} + +func (iv interval32) equal(b interval32) bool { + if iv.start == b.start { + return iv.last == b.last + } + return false +} + +func (iv interval32) isSuperSetOf(b interval32) bool { + return iv.start <= b.start && b.last <= iv.last +} + +func (iv interval32) subtractInterval(del interval32) (left []interval32, delcount int64) { + isect, isEmpty := intersectInterval32s(iv, del) + + if isEmpty { + return nil, 0 + } + if del.isSuperSetOf(iv) { + return nil, iv.runlen() + } + + switch { + case isect.start > iv.start && isect.last < iv.last: + new0 := interval32{start: iv.start, last: isect.start - 1} + new1 := interval32{start: isect.last + 1, last: iv.last} + return []interval32{new0, new1}, isect.runlen() + case isect.start == iv.start: + return []interval32{{start: isect.last + 1, last: iv.last}}, isect.runlen() + default: + return []interval32{{start: iv.start, last: isect.start - 1}}, isect.runlen() + } +} + +func (rc *runContainer32) isubtract(del interval32) { + origiv := make([]interval32, len(rc.iv)) + copy(origiv, rc.iv) + n := int64(len(rc.iv)) + if n == 0 { + return // already done. + } + + _, isEmpty := intersectInterval32s( + interval32{ + start: rc.iv[0].start, + last: rc.iv[n-1].last, + }, del) + if isEmpty { + return // done + } + // INVAR there is some intersection between rc and del + istart, startAlready, _ := rc.search(int64(del.start), nil) + ilast, lastAlready, _ := rc.search(int64(del.last), nil) + rc.card = -1 + if istart == -1 { + if ilast == n-1 && !lastAlready { + rc.iv = nil + return + } + } + // some intervals will remain + switch { + case startAlready && lastAlready: + res0, _ := rc.iv[istart].subtractInterval(del) + + // would overwrite values in iv b/c res0 can have len 2. so + // write to origiv instead. + lost := 1 + ilast - istart + changeSize := int64(len(res0)) - lost + newSize := int64(len(rc.iv)) + changeSize + + // rc.iv = append(pre, caboose...) + // return + + if ilast != istart { + res1, _ := rc.iv[ilast].subtractInterval(del) + res0 = append(res0, res1...) + changeSize = int64(len(res0)) - lost + newSize = int64(len(rc.iv)) + changeSize + } + switch { + case changeSize < 0: + // shrink + copy(rc.iv[istart+int64(len(res0)):], rc.iv[ilast+1:]) + copy(rc.iv[istart:istart+int64(len(res0))], res0) + rc.iv = rc.iv[:newSize] + return + case changeSize == 0: + // stay the same + copy(rc.iv[istart:istart+int64(len(res0))], res0) + return + default: + // changeSize > 0 is only possible when ilast == istart. + // Hence we now know: changeSize == 1 and len(res0) == 2 + rc.iv = append(rc.iv, interval32{}) + // len(rc.iv) is correct now, no need to rc.iv = rc.iv[:newSize] + + // copy the tail into place + copy(rc.iv[ilast+2:], rc.iv[ilast+1:]) + // copy the new item(s) into place + copy(rc.iv[istart:istart+2], res0) + return + } + + case !startAlready && !lastAlready: + // we get to discard whole intervals + + // from the search() definition: + + // if del.start is not present, then istart is + // set as follows: + // + // a) istart == n-1 if del.start is beyond our + // last interval32 in rc.iv; + // + // b) istart == -1 if del.start is before our first + // interval32 in rc.iv; + // + // c) istart is set to the minimum index of rc.iv + // which comes strictly before the del.start; + // so del.start > rc.iv[istart].last, + // and if istart+1 exists, then del.start < rc.iv[istart+1].startx + + // if del.last is not present, then ilast is + // set as follows: + // + // a) ilast == n-1 if del.last is beyond our + // last interval32 in rc.iv; + // + // b) ilast == -1 if del.last is before our first + // interval32 in rc.iv; + // + // c) ilast is set to the minimum index of rc.iv + // which comes strictly before the del.last; + // so del.last > rc.iv[ilast].last, + // and if ilast+1 exists, then del.last < rc.iv[ilast+1].start + + // INVAR: istart >= 0 + pre := rc.iv[:istart+1] + if ilast == n-1 { + rc.iv = pre + return + } + // INVAR: ilast < n-1 + lost := ilast - istart + changeSize := -lost + newSize := int64(len(rc.iv)) + changeSize + if changeSize != 0 { + copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) + } + rc.iv = rc.iv[:newSize] + return + + case startAlready && !lastAlready: + // we can only shrink or stay the same size + // i.e. we either eliminate the whole interval, + // or just cut off the right side. + res0, _ := rc.iv[istart].subtractInterval(del) + if len(res0) > 0 { + // len(res) must be 1 + rc.iv[istart] = res0[0] + } + lost := 1 + (ilast - istart) + changeSize := int64(len(res0)) - lost + newSize := int64(len(rc.iv)) + changeSize + if changeSize != 0 { + copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) + } + rc.iv = rc.iv[:newSize] + return + + case !startAlready && lastAlready: + // we can only shrink or stay the same size + res1, _ := rc.iv[ilast].subtractInterval(del) + lost := ilast - istart + changeSize := int64(len(res1)) - lost + newSize := int64(len(rc.iv)) + changeSize + if changeSize != 0 { + // move the tail first to make room for res1 + copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) + } + copy(rc.iv[istart+1:], res1) + rc.iv = rc.iv[:newSize] + return + } +} + +// compute rc minus b, and return the result as a new value (not inplace). +// port of run_container_andnot from CRoaring... +// https://github.com/RoaringBitmap/CRoaring/blob/master/src/containers/run.c#L435-L496 +func (rc *runContainer32) AndNotRunContainer32(b *runContainer32) *runContainer32 { + + if len(b.iv) == 0 || len(rc.iv) == 0 { + return rc + } + + dst := newRunContainer32() + apos := 0 + bpos := 0 + + a := rc + + astart := a.iv[apos].start + alast := a.iv[apos].last + bstart := b.iv[bpos].start + blast := b.iv[bpos].last + + alen := len(a.iv) + blen := len(b.iv) + + for apos < alen && bpos < blen { + switch { + case alast < bstart: + // output the first run + dst.iv = append(dst.iv, interval32{start: astart, last: alast}) + apos++ + if apos < alen { + astart = a.iv[apos].start + alast = a.iv[apos].last + } + case blast < astart: + // exit the second run + bpos++ + if bpos < blen { + bstart = b.iv[bpos].start + blast = b.iv[bpos].last + } + default: + // a: [ ] + // b: [ ] + // alast >= bstart + // blast >= astart + if astart < bstart { + dst.iv = append(dst.iv, interval32{start: astart, last: bstart - 1}) + } + if alast > blast { + astart = blast + 1 + } else { + apos++ + if apos < alen { + astart = a.iv[apos].start + alast = a.iv[apos].last + } + } + } + } + if apos < alen { + dst.iv = append(dst.iv, interval32{start: astart, last: alast}) + apos++ + if apos < alen { + dst.iv = append(dst.iv, a.iv[apos:]...) + } + } + + return dst +} + +func (rc *runContainer32) numberOfRuns() (nr int) { + return len(rc.iv) +} + +func (rc *runContainer32) containerType() contype { + return run32Contype +} + +func (rc *runContainer32) equals32(srb *runContainer32) bool { + //p("both rc32") + // Check if the containers are the same object. + if rc == srb { + //p("same object") + return true + } + + if len(srb.iv) != len(rc.iv) { + //p("iv len differ") + return false + } + + for i, v := range rc.iv { + if v != srb.iv[i] { + //p("differ at iv i=%v, srb.iv[i]=%v, rc.iv[i]=%v", i, srb.iv[i], rc.iv[i]) + return false + } + } + //p("all intervals same, returning true") + return true +} diff --git a/vendor/github.com/RoaringBitmap/roaring/rle16.go b/vendor/github.com/RoaringBitmap/roaring/rle16.go new file mode 100644 index 00000000..951af65f --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/rle16.go @@ -0,0 +1,1747 @@ +package roaring + +// +// Copyright (c) 2016 by the roaring authors. +// Licensed under the Apache License, Version 2.0. +// +// We derive a few lines of code from the sort.Search +// function in the golang standard library. That function +// is Copyright 2009 The Go Authors, and licensed +// under the following BSD-style license. +/* +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +import ( + "fmt" + "sort" + "unsafe" +) + +//go:generate msgp -unexported + +// runContainer16 does run-length encoding of sets of +// uint16 integers. +type runContainer16 struct { + iv []interval16 + card int64 + + // avoid allocation during search + myOpts searchOptions `msg:"-"` +} + +// interval16 is the internal to runContainer16 +// structure that maintains the individual [start, last] +// closed intervals. +type interval16 struct { + start uint16 + length uint16 // length minus 1 +} + +func newInterval16Range(start, last uint16) interval16 { + if last < start { + panic(fmt.Sprintf("last (%d) cannot be smaller than start (%d)", last, start)) + } + + return interval16{ + start, + last - start, + } +} + +// runlen returns the count of integers in the interval. +func (iv interval16) runlen() int64 { + return int64(iv.length) + 1 +} + +func (iv interval16) last() uint16 { + return iv.start + iv.length +} + +// String produces a human viewable string of the contents. +func (iv interval16) String() string { + return fmt.Sprintf("[%d, %d]", iv.start, iv.length) +} + +func ivalString16(iv []interval16) string { + var s string + var j int + var p interval16 + for j, p = range iv { + s += fmt.Sprintf("%v:[%d, %d], ", j, p.start, p.last()) + } + return s +} + +// String produces a human viewable string of the contents. +func (rc *runContainer16) String() string { + if len(rc.iv) == 0 { + return "runContainer16{}" + } + is := ivalString16(rc.iv) + return `runContainer16{` + is + `}` +} + +// uint16Slice is a sort.Sort convenience method +type uint16Slice []uint16 + +// Len returns the length of p. +func (p uint16Slice) Len() int { return len(p) } + +// Less returns p[i] < p[j] +func (p uint16Slice) Less(i, j int) bool { return p[i] < p[j] } + +// Swap swaps elements i and j. +func (p uint16Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +//msgp:ignore addHelper + +// addHelper helps build a runContainer16. +type addHelper16 struct { + runstart uint16 + runlen uint16 + actuallyAdded uint16 + m []interval16 + rc *runContainer16 +} + +func (ah *addHelper16) storeIval(runstart, runlen uint16) { + mi := interval16{start: runstart, length: runlen} + ah.m = append(ah.m, mi) +} + +func (ah *addHelper16) add(cur, prev uint16, i int) { + if cur == prev+1 { + ah.runlen++ + ah.actuallyAdded++ + } else { + if cur < prev { + panic(fmt.Sprintf("newRunContainer16FromVals sees "+ + "unsorted vals; vals[%v]=cur=%v < prev=%v. Sort your vals"+ + " before calling us with alreadySorted == true.", i, cur, prev)) + } + if cur == prev { + // ignore duplicates + } else { + ah.actuallyAdded++ + ah.storeIval(ah.runstart, ah.runlen) + ah.runstart = cur + ah.runlen = 0 + } + } +} + +// newRunContainerRange makes a new container made of just the specified closed interval [rangestart,rangelast] +func newRunContainer16Range(rangestart uint16, rangelast uint16) *runContainer16 { + rc := &runContainer16{} + rc.iv = append(rc.iv, newInterval16Range(rangestart, rangelast)) + return rc +} + +// newRunContainer16FromVals makes a new container from vals. +// +// For efficiency, vals should be sorted in ascending order. +// Ideally vals should not contain duplicates, but we detect and +// ignore them. If vals is already sorted in ascending order, then +// pass alreadySorted = true. Otherwise, for !alreadySorted, +// we will sort vals before creating a runContainer16 of them. +// We sort the original vals, so this will change what the +// caller sees in vals as a side effect. +func newRunContainer16FromVals(alreadySorted bool, vals ...uint16) *runContainer16 { + // keep this in sync with newRunContainer16FromArray below + + rc := &runContainer16{} + ah := addHelper16{rc: rc} + + if !alreadySorted { + sort.Sort(uint16Slice(vals)) + } + n := len(vals) + var cur, prev uint16 + switch { + case n == 0: + // nothing more + case n == 1: + ah.m = append(ah.m, newInterval16Range(vals[0], vals[0])) + ah.actuallyAdded++ + default: + ah.runstart = vals[0] + ah.actuallyAdded++ + for i := 1; i < n; i++ { + prev = vals[i-1] + cur = vals[i] + ah.add(cur, prev, i) + } + ah.storeIval(ah.runstart, ah.runlen) + } + rc.iv = ah.m + rc.card = int64(ah.actuallyAdded) + return rc +} + +// newRunContainer16FromBitmapContainer makes a new run container from bc, +// somewhat efficiently. For reference, see the Java +// https://github.com/RoaringBitmap/RoaringBitmap/blob/master/src/main/java/org/roaringbitmap/RunContainer.java#L145-L192 +func newRunContainer16FromBitmapContainer(bc *bitmapContainer) *runContainer16 { + + rc := &runContainer16{} + nbrRuns := bc.numberOfRuns() + if nbrRuns == 0 { + return rc + } + rc.iv = make([]interval16, nbrRuns) + + longCtr := 0 // index of current long in bitmap + curWord := bc.bitmap[0] // its value + runCount := 0 + for { + // potentially multiword advance to first 1 bit + for curWord == 0 && longCtr < len(bc.bitmap)-1 { + longCtr++ + curWord = bc.bitmap[longCtr] + } + + if curWord == 0 { + // wrap up, no more runs + return rc + } + localRunStart := countTrailingZeros(curWord) + runStart := localRunStart + 64*longCtr + // stuff 1s into number's LSBs + curWordWith1s := curWord | (curWord - 1) + + // find the next 0, potentially in a later word + runEnd := 0 + for curWordWith1s == maxWord && longCtr < len(bc.bitmap)-1 { + longCtr++ + curWordWith1s = bc.bitmap[longCtr] + } + + if curWordWith1s == maxWord { + // a final unterminated run of 1s + runEnd = wordSizeInBits + longCtr*64 + rc.iv[runCount].start = uint16(runStart) + rc.iv[runCount].length = uint16(runEnd) - uint16(runStart) - 1 + return rc + } + localRunEnd := countTrailingZeros(^curWordWith1s) + runEnd = localRunEnd + longCtr*64 + rc.iv[runCount].start = uint16(runStart) + rc.iv[runCount].length = uint16(runEnd) - 1 - uint16(runStart) + runCount++ + // now, zero out everything right of runEnd. + curWord = curWordWith1s & (curWordWith1s + 1) + // We've lathered and rinsed, so repeat... + } + +} + +// +// newRunContainer16FromArray populates a new +// runContainer16 from the contents of arr. +// +func newRunContainer16FromArray(arr *arrayContainer) *runContainer16 { + // keep this in sync with newRunContainer16FromVals above + + rc := &runContainer16{} + ah := addHelper16{rc: rc} + + n := arr.getCardinality() + var cur, prev uint16 + switch { + case n == 0: + // nothing more + case n == 1: + ah.m = append(ah.m, newInterval16Range(arr.content[0], arr.content[0])) + ah.actuallyAdded++ + default: + ah.runstart = arr.content[0] + ah.actuallyAdded++ + for i := 1; i < n; i++ { + prev = arr.content[i-1] + cur = arr.content[i] + ah.add(cur, prev, i) + } + ah.storeIval(ah.runstart, ah.runlen) + } + rc.iv = ah.m + rc.card = int64(ah.actuallyAdded) + return rc +} + +// set adds the integers in vals to the set. Vals +// must be sorted in increasing order; if not, you should set +// alreadySorted to false, and we will sort them in place for you. +// (Be aware of this side effect -- it will affect the callers +// view of vals). +// +// If you have a small number of additions to an already +// big runContainer16, calling Add() may be faster. +func (rc *runContainer16) set(alreadySorted bool, vals ...uint16) { + + rc2 := newRunContainer16FromVals(alreadySorted, vals...) + un := rc.union(rc2) + rc.iv = un.iv + rc.card = 0 +} + +// canMerge returns true iff the intervals +// a and b either overlap or they are +// contiguous and so can be merged into +// a single interval. +func canMerge16(a, b interval16) bool { + if int64(a.last())+1 < int64(b.start) { + return false + } + return int64(b.last())+1 >= int64(a.start) +} + +// haveOverlap differs from canMerge in that +// it tells you if the intersection of a +// and b would contain an element (otherwise +// it would be the empty set, and we return +// false). +func haveOverlap16(a, b interval16) bool { + if int64(a.last())+1 <= int64(b.start) { + return false + } + return int64(b.last())+1 > int64(a.start) +} + +// mergeInterval16s joins a and b into a +// new interval, and panics if it cannot. +func mergeInterval16s(a, b interval16) (res interval16) { + if !canMerge16(a, b) { + panic(fmt.Sprintf("cannot merge %#v and %#v", a, b)) + } + + if b.start < a.start { + res.start = b.start + } else { + res.start = a.start + } + + if b.last() > a.last() { + res.length = b.last() - res.start + } else { + res.length = a.last() - res.start + } + + return +} + +// intersectInterval16s returns the intersection +// of a and b. The isEmpty flag will be true if +// a and b were disjoint. +func intersectInterval16s(a, b interval16) (res interval16, isEmpty bool) { + if !haveOverlap16(a, b) { + isEmpty = true + return + } + if b.start > a.start { + res.start = b.start + } else { + res.start = a.start + } + + bEnd := b.last() + aEnd := a.last() + var resEnd uint16 + + if bEnd < aEnd { + resEnd = bEnd + } else { + resEnd = aEnd + } + res.length = resEnd - res.start + return +} + +// union merges two runContainer16s, producing +// a new runContainer16 with the union of rc and b. +func (rc *runContainer16) union(b *runContainer16) *runContainer16 { + + // rc is also known as 'a' here, but golint insisted we + // call it rc for consistency with the rest of the methods. + + var m []interval16 + + alim := int64(len(rc.iv)) + blim := int64(len(b.iv)) + + var na int64 // next from a + var nb int64 // next from b + + // merged holds the current merge output, which might + // get additional merges before being appended to m. + var merged interval16 + var mergedUsed bool // is merged being used at the moment? + + var cura interval16 // currently considering this interval16 from a + var curb interval16 // currently considering this interval16 from b + + pass := 0 + for na < alim && nb < blim { + pass++ + cura = rc.iv[na] + curb = b.iv[nb] + + if mergedUsed { + mergedUpdated := false + if canMerge16(cura, merged) { + merged = mergeInterval16s(cura, merged) + na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + mergedUpdated = true + } + if canMerge16(curb, merged) { + merged = mergeInterval16s(curb, merged) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + mergedUpdated = true + } + if !mergedUpdated { + // we know that merged is disjoint from cura and curb + m = append(m, merged) + mergedUsed = false + } + continue + + } else { + // !mergedUsed + if !canMerge16(cura, curb) { + if cura.start < curb.start { + m = append(m, cura) + na++ + } else { + m = append(m, curb) + nb++ + } + } else { + merged = mergeInterval16s(cura, curb) + mergedUsed = true + na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + } + } + } + var aDone, bDone bool + if na >= alim { + aDone = true + } + if nb >= blim { + bDone = true + } + // finish by merging anything remaining into merged we can: + if mergedUsed { + if !aDone { + aAdds: + for na < alim { + cura = rc.iv[na] + if canMerge16(cura, merged) { + merged = mergeInterval16s(cura, merged) + na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + } else { + break aAdds + } + } + + } + + if !bDone { + bAdds: + for nb < blim { + curb = b.iv[nb] + if canMerge16(curb, merged) { + merged = mergeInterval16s(curb, merged) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + } else { + break bAdds + } + } + + } + + m = append(m, merged) + } + if na < alim { + m = append(m, rc.iv[na:]...) + } + if nb < blim { + m = append(m, b.iv[nb:]...) + } + + res := &runContainer16{iv: m} + return res +} + +// unionCardinality returns the cardinality of the merger of two runContainer16s, the union of rc and b. +func (rc *runContainer16) unionCardinality(b *runContainer16) uint64 { + + // rc is also known as 'a' here, but golint insisted we + // call it rc for consistency with the rest of the methods. + answer := uint64(0) + + alim := int64(len(rc.iv)) + blim := int64(len(b.iv)) + + var na int64 // next from a + var nb int64 // next from b + + // merged holds the current merge output, which might + // get additional merges before being appended to m. + var merged interval16 + var mergedUsed bool // is merged being used at the moment? + + var cura interval16 // currently considering this interval16 from a + var curb interval16 // currently considering this interval16 from b + + pass := 0 + for na < alim && nb < blim { + pass++ + cura = rc.iv[na] + curb = b.iv[nb] + + if mergedUsed { + mergedUpdated := false + if canMerge16(cura, merged) { + merged = mergeInterval16s(cura, merged) + na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + mergedUpdated = true + } + if canMerge16(curb, merged) { + merged = mergeInterval16s(curb, merged) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + mergedUpdated = true + } + if !mergedUpdated { + // we know that merged is disjoint from cura and curb + //m = append(m, merged) + answer += uint64(merged.last()) - uint64(merged.start) + 1 + mergedUsed = false + } + continue + + } else { + // !mergedUsed + if !canMerge16(cura, curb) { + if cura.start < curb.start { + answer += uint64(cura.last()) - uint64(cura.start) + 1 + //m = append(m, cura) + na++ + } else { + answer += uint64(curb.last()) - uint64(curb.start) + 1 + //m = append(m, curb) + nb++ + } + } else { + merged = mergeInterval16s(cura, curb) + mergedUsed = true + na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + } + } + } + var aDone, bDone bool + if na >= alim { + aDone = true + } + if nb >= blim { + bDone = true + } + // finish by merging anything remaining into merged we can: + if mergedUsed { + if !aDone { + aAdds: + for na < alim { + cura = rc.iv[na] + if canMerge16(cura, merged) { + merged = mergeInterval16s(cura, merged) + na = rc.indexOfIntervalAtOrAfter(int64(merged.last())+1, na+1) + } else { + break aAdds + } + } + + } + + if !bDone { + bAdds: + for nb < blim { + curb = b.iv[nb] + if canMerge16(curb, merged) { + merged = mergeInterval16s(curb, merged) + nb = b.indexOfIntervalAtOrAfter(int64(merged.last())+1, nb+1) + } else { + break bAdds + } + } + + } + + //m = append(m, merged) + answer += uint64(merged.last()) - uint64(merged.start) + 1 + } + for _, r := range rc.iv[na:] { + answer += uint64(r.last()) - uint64(r.start) + 1 + } + for _, r := range b.iv[nb:] { + answer += uint64(r.last()) - uint64(r.start) + 1 + } + return answer +} + +// indexOfIntervalAtOrAfter is a helper for union. +func (rc *runContainer16) indexOfIntervalAtOrAfter(key int64, startIndex int64) int64 { + rc.myOpts.startIndex = startIndex + rc.myOpts.endxIndex = 0 + + w, already, _ := rc.search(key, &rc.myOpts) + if already { + return w + } + return w + 1 +} + +// intersect returns a new runContainer16 holding the +// intersection of rc (also known as 'a') and b. +func (rc *runContainer16) intersect(b *runContainer16) *runContainer16 { + + a := rc + numa := int64(len(a.iv)) + numb := int64(len(b.iv)) + res := &runContainer16{} + if numa == 0 || numb == 0 { + return res + } + + if numa == 1 && numb == 1 { + if !haveOverlap16(a.iv[0], b.iv[0]) { + return res + } + } + + var output []interval16 + + var acuri int64 + var bcuri int64 + + astart := int64(a.iv[acuri].start) + bstart := int64(b.iv[bcuri].start) + + var intersection interval16 + var leftoverstart int64 + var isOverlap, isLeftoverA, isLeftoverB bool + var done bool +toploop: + for acuri < numa && bcuri < numb { + + isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = + intersectWithLeftover16(astart, int64(a.iv[acuri].last()), bstart, int64(b.iv[bcuri].last())) + + if !isOverlap { + switch { + case astart < bstart: + acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart) + if done { + break toploop + } + astart = int64(a.iv[acuri].start) + + case astart > bstart: + bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) + if done { + break toploop + } + bstart = int64(b.iv[bcuri].start) + + //default: + // panic("impossible that astart == bstart, since !isOverlap") + } + + } else { + // isOverlap + output = append(output, intersection) + switch { + case isLeftoverA: + // note that we change astart without advancing acuri, + // since we need to capture any 2ndary intersections with a.iv[acuri] + astart = leftoverstart + bcuri++ + if bcuri >= numb { + break toploop + } + bstart = int64(b.iv[bcuri].start) + case isLeftoverB: + // note that we change bstart without advancing bcuri, + // since we need to capture any 2ndary intersections with b.iv[bcuri] + bstart = leftoverstart + acuri++ + if acuri >= numa { + break toploop + } + astart = int64(a.iv[acuri].start) + default: + // neither had leftover, both completely consumed + // optionally, assert for sanity: + //if a.iv[acuri].endx != b.iv[bcuri].endx { + // panic("huh? should only be possible that endx agree now!") + //} + + // advance to next a interval + acuri++ + if acuri >= numa { + break toploop + } + astart = int64(a.iv[acuri].start) + + // advance to next b interval + bcuri++ + if bcuri >= numb { + break toploop + } + bstart = int64(b.iv[bcuri].start) + } + } + } // end for toploop + + if len(output) == 0 { + return res + } + + res.iv = output + return res +} + +// intersectCardinality returns the cardinality of the +// intersection of rc (also known as 'a') and b. +func (rc *runContainer16) intersectCardinality(b *runContainer16) int64 { + answer := int64(0) + + a := rc + numa := int64(len(a.iv)) + numb := int64(len(b.iv)) + if numa == 0 || numb == 0 { + return 0 + } + + if numa == 1 && numb == 1 { + if !haveOverlap16(a.iv[0], b.iv[0]) { + return 0 + } + } + + var acuri int64 + var bcuri int64 + + astart := int64(a.iv[acuri].start) + bstart := int64(b.iv[bcuri].start) + + var intersection interval16 + var leftoverstart int64 + var isOverlap, isLeftoverA, isLeftoverB bool + var done bool + pass := 0 +toploop: + for acuri < numa && bcuri < numb { + pass++ + + isOverlap, isLeftoverA, isLeftoverB, leftoverstart, intersection = + intersectWithLeftover16(astart, int64(a.iv[acuri].last()), bstart, int64(b.iv[bcuri].last())) + + if !isOverlap { + switch { + case astart < bstart: + acuri, done = a.findNextIntervalThatIntersectsStartingFrom(acuri+1, bstart) + if done { + break toploop + } + astart = int64(a.iv[acuri].start) + + case astart > bstart: + bcuri, done = b.findNextIntervalThatIntersectsStartingFrom(bcuri+1, astart) + if done { + break toploop + } + bstart = int64(b.iv[bcuri].start) + + //default: + // panic("impossible that astart == bstart, since !isOverlap") + } + + } else { + // isOverlap + answer += int64(intersection.last()) - int64(intersection.start) + 1 + switch { + case isLeftoverA: + // note that we change astart without advancing acuri, + // since we need to capture any 2ndary intersections with a.iv[acuri] + astart = leftoverstart + bcuri++ + if bcuri >= numb { + break toploop + } + bstart = int64(b.iv[bcuri].start) + case isLeftoverB: + // note that we change bstart without advancing bcuri, + // since we need to capture any 2ndary intersections with b.iv[bcuri] + bstart = leftoverstart + acuri++ + if acuri >= numa { + break toploop + } + astart = int64(a.iv[acuri].start) + default: + // neither had leftover, both completely consumed + // optionally, assert for sanity: + //if a.iv[acuri].endx != b.iv[bcuri].endx { + // panic("huh? should only be possible that endx agree now!") + //} + + // advance to next a interval + acuri++ + if acuri >= numa { + break toploop + } + astart = int64(a.iv[acuri].start) + + // advance to next b interval + bcuri++ + if bcuri >= numb { + break toploop + } + bstart = int64(b.iv[bcuri].start) + } + } + } // end for toploop + + return answer +} + +// get returns true iff key is in the container. +func (rc *runContainer16) contains(key uint16) bool { + _, in, _ := rc.search(int64(key), nil) + return in +} + +// numIntervals returns the count of intervals in the container. +func (rc *runContainer16) numIntervals() int { + return len(rc.iv) +} + +// search returns alreadyPresent to indicate if the +// key is already in one of our interval16s. +// +// If key is alreadyPresent, then whichInterval16 tells +// you where. +// +// If key is not already present, then whichInterval16 is +// set as follows: +// +// a) whichInterval16 == len(rc.iv)-1 if key is beyond our +// last interval16 in rc.iv; +// +// b) whichInterval16 == -1 if key is before our first +// interval16 in rc.iv; +// +// c) whichInterval16 is set to the minimum index of rc.iv +// which comes strictly before the key; +// so rc.iv[whichInterval16].last < key, +// and if whichInterval16+1 exists, then key < rc.iv[whichInterval16+1].start +// (Note that whichInterval16+1 won't exist when +// whichInterval16 is the last interval.) +// +// runContainer16.search always returns whichInterval16 < len(rc.iv). +// +// If not nil, opts can be used to further restrict +// the search space. +// +func (rc *runContainer16) search(key int64, opts *searchOptions) (whichInterval16 int64, alreadyPresent bool, numCompares int) { + n := int64(len(rc.iv)) + if n == 0 { + return -1, false, 0 + } + + startIndex := int64(0) + endxIndex := n + if opts != nil { + startIndex = opts.startIndex + + // let endxIndex == 0 mean no effect + if opts.endxIndex > 0 { + endxIndex = opts.endxIndex + } + } + + // sort.Search returns the smallest index i + // in [0, n) at which f(i) is true, assuming that on the range [0, n), + // f(i) == true implies f(i+1) == true. + // If there is no such index, Search returns n. + + // For correctness, this began as verbatim snippet from + // sort.Search in the Go standard lib. + // We inline our comparison function for speed, and + // annotate with numCompares + // to observe and test that extra bounds are utilized. + i, j := startIndex, endxIndex + for i < j { + h := i + (j-i)/2 // avoid overflow when computing h as the bisector + // i <= h < j + numCompares++ + if !(key < int64(rc.iv[h].start)) { + i = h + 1 + } else { + j = h + } + } + below := i + // end std lib snippet. + + // The above is a simple in-lining and annotation of: + /* below := sort.Search(n, + func(i int) bool { + return key < rc.iv[i].start + }) + */ + whichInterval16 = below - 1 + + if below == n { + // all falses => key is >= start of all interval16s + // ... so does it belong to the last interval16? + if key < int64(rc.iv[n-1].last())+1 { + // yes, it belongs to the last interval16 + alreadyPresent = true + return + } + // no, it is beyond the last interval16. + // leave alreadyPreset = false + return + } + + // INVAR: key is below rc.iv[below] + if below == 0 { + // key is before the first first interval16. + // leave alreadyPresent = false + return + } + + // INVAR: key is >= rc.iv[below-1].start and + // key is < rc.iv[below].start + + // is key in below-1 interval16? + if key >= int64(rc.iv[below-1].start) && key < int64(rc.iv[below-1].last())+1 { + // yes, it is. key is in below-1 interval16. + alreadyPresent = true + return + } + + // INVAR: key >= rc.iv[below-1].endx && key < rc.iv[below].start + // leave alreadyPresent = false + return +} + +// cardinality returns the count of the integers stored in the +// runContainer16. +func (rc *runContainer16) cardinality() int64 { + if len(rc.iv) == 0 { + rc.card = 0 + return 0 + } + if rc.card > 0 { + return rc.card // already cached + } + // have to compute it + var n int64 + for _, p := range rc.iv { + n += p.runlen() + } + rc.card = n // cache it + return n +} + +// AsSlice decompresses the contents into a []uint16 slice. +func (rc *runContainer16) AsSlice() []uint16 { + s := make([]uint16, rc.cardinality()) + j := 0 + for _, p := range rc.iv { + for i := p.start; i <= p.last(); i++ { + s[j] = i + j++ + } + } + return s +} + +// newRunContainer16 creates an empty run container. +func newRunContainer16() *runContainer16 { + return &runContainer16{} +} + +// newRunContainer16CopyIv creates a run container, initializing +// with a copy of the supplied iv slice. +// +func newRunContainer16CopyIv(iv []interval16) *runContainer16 { + rc := &runContainer16{ + iv: make([]interval16, len(iv)), + } + copy(rc.iv, iv) + return rc +} + +func (rc *runContainer16) Clone() *runContainer16 { + rc2 := newRunContainer16CopyIv(rc.iv) + return rc2 +} + +// newRunContainer16TakeOwnership returns a new runContainer16 +// backed by the provided iv slice, which we will +// assume exclusive control over from now on. +// +func newRunContainer16TakeOwnership(iv []interval16) *runContainer16 { + rc := &runContainer16{ + iv: iv, + } + return rc +} + +const baseRc16Size = int(unsafe.Sizeof(runContainer16{})) +const perIntervalRc16Size = int(unsafe.Sizeof(interval16{})) + +const baseDiskRc16Size = int(unsafe.Sizeof(uint16(0))) + +// see also runContainer16SerializedSizeInBytes(numRuns int) int + +// getSizeInBytes returns the number of bytes of memory +// required by this runContainer16. +func (rc *runContainer16) getSizeInBytes() int { + return perIntervalRc16Size*len(rc.iv) + baseRc16Size +} + +// runContainer16SerializedSizeInBytes returns the number of bytes of disk +// required to hold numRuns in a runContainer16. +func runContainer16SerializedSizeInBytes(numRuns int) int { + return perIntervalRc16Size*numRuns + baseDiskRc16Size +} + +// Add adds a single value k to the set. +func (rc *runContainer16) Add(k uint16) (wasNew bool) { + // TODO comment from runContainer16.java: + // it might be better and simpler to do return + // toBitmapOrArrayContainer(getCardinality()).add(k) + // but note that some unit tests use this method to build up test + // runcontainers without calling runOptimize + + k64 := int64(k) + + index, present, _ := rc.search(k64, nil) + if present { + return // already there + } + wasNew = true + + // increment card if it is cached already + if rc.card > 0 { + rc.card++ + } + n := int64(len(rc.iv)) + if index == -1 { + // we may need to extend the first run + if n > 0 { + if rc.iv[0].start == k+1 { + rc.iv[0].start = k + rc.iv[0].length++ + return + } + } + // nope, k stands alone, starting the new first interval16. + rc.iv = append([]interval16{newInterval16Range(k, k)}, rc.iv...) + return + } + + // are we off the end? handle both index == n and index == n-1: + if index >= n-1 { + if int64(rc.iv[n-1].last())+1 == k64 { + rc.iv[n-1].length++ + return + } + rc.iv = append(rc.iv, newInterval16Range(k, k)) + return + } + + // INVAR: index and index+1 both exist, and k goes between them. + // + // Now: add k into the middle, + // possibly fusing with index or index+1 interval16 + // and possibly resulting in fusing of two interval16s + // that had a one integer gap. + + left := index + right := index + 1 + + // are we fusing left and right by adding k? + if int64(rc.iv[left].last())+1 == k64 && int64(rc.iv[right].start) == k64+1 { + // fuse into left + rc.iv[left].length = rc.iv[right].last() - rc.iv[left].start + // remove redundant right + rc.iv = append(rc.iv[:left+1], rc.iv[right+1:]...) + return + } + + // are we an addition to left? + if int64(rc.iv[left].last())+1 == k64 { + // yes + rc.iv[left].length++ + return + } + + // are we an addition to right? + if int64(rc.iv[right].start) == k64+1 { + // yes + rc.iv[right].start = k + rc.iv[right].length++ + return + } + + // k makes a standalone new interval16, inserted in the middle + tail := append([]interval16{newInterval16Range(k, k)}, rc.iv[right:]...) + rc.iv = append(rc.iv[:left+1], tail...) + return +} + +//msgp:ignore runIterator + +// runIterator16 advice: you must call Next() at least once +// before calling Cur(); and you should call HasNext() +// before calling Next() to insure there are contents. +type runIterator16 struct { + rc *runContainer16 + curIndex int64 + curPosInIndex uint16 + curSeq int64 +} + +// newRunIterator16 returns a new empty run container. +func (rc *runContainer16) newRunIterator16() *runIterator16 { + return &runIterator16{rc: rc, curIndex: -1} +} + +// HasNext returns false if calling Next will panic. It +// returns true when there is at least one more value +// available in the iteration sequence. +func (ri *runIterator16) hasNext() bool { + if len(ri.rc.iv) == 0 { + return false + } + if ri.curIndex == -1 { + return true + } + return ri.curSeq+1 < ri.rc.cardinality() +} + +// cur returns the current value pointed to by the iterator. +func (ri *runIterator16) cur() uint16 { + return ri.rc.iv[ri.curIndex].start + ri.curPosInIndex +} + +// Next returns the next value in the iteration sequence. +func (ri *runIterator16) next() uint16 { + if !ri.hasNext() { + panic("no Next available") + } + if ri.curIndex >= int64(len(ri.rc.iv)) { + panic("runIterator.Next() going beyond what is available") + } + if ri.curIndex == -1 { + // first time is special + ri.curIndex = 0 + } else { + ri.curPosInIndex++ + if int64(ri.rc.iv[ri.curIndex].start)+int64(ri.curPosInIndex) == int64(ri.rc.iv[ri.curIndex].last())+1 { + ri.curPosInIndex = 0 + ri.curIndex++ + } + ri.curSeq++ + } + return ri.cur() +} + +// remove removes the element that the iterator +// is on from the run container. You can use +// Cur if you want to double check what is about +// to be deleted. +func (ri *runIterator16) remove() uint16 { + n := ri.rc.cardinality() + if n == 0 { + panic("runIterator.Remove called on empty runContainer16") + } + cur := ri.cur() + + ri.rc.deleteAt(&ri.curIndex, &ri.curPosInIndex, &ri.curSeq) + return cur +} + +type manyRunIterator16 struct { + rc *runContainer16 + curIndex int64 + curPosInIndex uint16 + curSeq int64 +} + +func (rc *runContainer16) newManyRunIterator16() *manyRunIterator16 { + return &manyRunIterator16{rc: rc, curIndex: -1} +} + +func (ri *manyRunIterator16) hasNext() bool { + if len(ri.rc.iv) == 0 { + return false + } + if ri.curIndex == -1 { + return true + } + return ri.curSeq+1 < ri.rc.cardinality() +} + +// hs are the high bits to include to avoid needing to reiterate over the buffer in NextMany +func (ri *manyRunIterator16) nextMany(hs uint32, buf []uint32) int { + n := 0 + if !ri.hasNext() { + return n + } + // start and end are inclusive + for n < len(buf) { + if ri.curIndex == -1 || int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex) <= 0 { + ri.curPosInIndex = 0 + ri.curIndex++ + if ri.curIndex == int64(len(ri.rc.iv)) { + break + } + buf[n] = uint32(ri.rc.iv[ri.curIndex].start) | hs + if ri.curIndex != 0 { + ri.curSeq += 1 + } + n += 1 + // not strictly necessarily due to len(buf)-n min check, but saves some work + continue + } + // add as many as you can from this seq + moreVals := minOfInt(int(ri.rc.iv[ri.curIndex].length-ri.curPosInIndex), len(buf)-n) + + base := uint32(ri.rc.iv[ri.curIndex].start+ri.curPosInIndex+1) | hs + + // allows BCE + buf2 := buf[n : n+moreVals] + for i := range buf2 { + buf2[i] = base + uint32(i) + } + + // update values + ri.curPosInIndex += uint16(moreVals) //moreVals always fits in uint16 + ri.curSeq += int64(moreVals) + n += moreVals + } + return n +} + +// remove removes key from the container. +func (rc *runContainer16) removeKey(key uint16) (wasPresent bool) { + + var index int64 + var curSeq int64 + index, wasPresent, _ = rc.search(int64(key), nil) + if !wasPresent { + return // already removed, nothing to do. + } + pos := key - rc.iv[index].start + rc.deleteAt(&index, &pos, &curSeq) + return +} + +// internal helper functions + +func (rc *runContainer16) deleteAt(curIndex *int64, curPosInIndex *uint16, curSeq *int64) { + rc.card-- + *curSeq-- + ci := *curIndex + pos := *curPosInIndex + + // are we first, last, or in the middle of our interval16? + switch { + case pos == 0: + if int64(rc.iv[ci].length) == 0 { + // our interval disappears + rc.iv = append(rc.iv[:ci], rc.iv[ci+1:]...) + // curIndex stays the same, since the delete did + // the advance for us. + *curPosInIndex = 0 + } else { + rc.iv[ci].start++ // no longer overflowable + rc.iv[ci].length-- + } + case pos == rc.iv[ci].length: + // length + rc.iv[ci].length-- + // our interval16 cannot disappear, else we would have been pos == 0, case first above. + *curPosInIndex-- + // if we leave *curIndex alone, then Next() will work properly even after the delete. + default: + //middle + // split into two, adding an interval16 + new0 := newInterval16Range(rc.iv[ci].start, rc.iv[ci].start+*curPosInIndex-1) + + new1start := int64(rc.iv[ci].start+*curPosInIndex) + 1 + if new1start > int64(MaxUint16) { + panic("overflow?!?!") + } + new1 := newInterval16Range(uint16(new1start), rc.iv[ci].last()) + tail := append([]interval16{new0, new1}, rc.iv[ci+1:]...) + rc.iv = append(rc.iv[:ci], tail...) + // update curIndex and curPosInIndex + *curIndex++ + *curPosInIndex = 0 + } + +} + +func have4Overlap16(astart, alast, bstart, blast int64) bool { + if alast+1 <= bstart { + return false + } + return blast+1 > astart +} + +func intersectWithLeftover16(astart, alast, bstart, blast int64) (isOverlap, isLeftoverA, isLeftoverB bool, leftoverstart int64, intersection interval16) { + if !have4Overlap16(astart, alast, bstart, blast) { + return + } + isOverlap = true + + // do the intersection: + if bstart > astart { + intersection.start = uint16(bstart) + } else { + intersection.start = uint16(astart) + } + + switch { + case blast < alast: + isLeftoverA = true + leftoverstart = blast + 1 + intersection.length = uint16(blast) - intersection.start + case alast < blast: + isLeftoverB = true + leftoverstart = alast + 1 + intersection.length = uint16(alast) - intersection.start + default: + // alast == blast + intersection.length = uint16(alast) - intersection.start + } + + return +} + +func (rc *runContainer16) findNextIntervalThatIntersectsStartingFrom(startIndex int64, key int64) (index int64, done bool) { + + rc.myOpts.startIndex = startIndex + rc.myOpts.endxIndex = 0 + + w, _, _ := rc.search(key, &rc.myOpts) + // rc.search always returns w < len(rc.iv) + if w < startIndex { + // not found and comes before lower bound startIndex, + // so just use the lower bound. + if startIndex == int64(len(rc.iv)) { + // also this bump up means that we are done + return startIndex, true + } + return startIndex, false + } + + return w, false +} + +func sliceToString16(m []interval16) string { + s := "" + for i := range m { + s += fmt.Sprintf("%v: %s, ", i, m[i]) + } + return s +} + +// selectInt16 returns the j-th value in the container. +// We panic of j is out of bounds. +func (rc *runContainer16) selectInt16(j uint16) int { + n := rc.cardinality() + if int64(j) > n { + panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n)) + } + + var offset int64 + for k := range rc.iv { + nextOffset := offset + rc.iv[k].runlen() + 1 + if nextOffset > int64(j) { + return int(int64(rc.iv[k].start) + (int64(j) - offset)) + } + offset = nextOffset + } + panic(fmt.Sprintf("Cannot select %v since Cardinality is %v", j, n)) +} + +// helper for invert +func (rc *runContainer16) invertlastInterval(origin uint16, lastIdx int) []interval16 { + cur := rc.iv[lastIdx] + if cur.last() == MaxUint16 { + if cur.start == origin { + return nil // empty container + } + return []interval16{newInterval16Range(origin, cur.start-1)} + } + if cur.start == origin { + return []interval16{newInterval16Range(cur.last()+1, MaxUint16)} + } + // invert splits + return []interval16{ + newInterval16Range(origin, cur.start-1), + newInterval16Range(cur.last()+1, MaxUint16), + } +} + +// invert returns a new container (not inplace), that is +// the inversion of rc. For each bit b in rc, the +// returned value has !b +func (rc *runContainer16) invert() *runContainer16 { + ni := len(rc.iv) + var m []interval16 + switch ni { + case 0: + return &runContainer16{iv: []interval16{newInterval16Range(0, MaxUint16)}} + case 1: + return &runContainer16{iv: rc.invertlastInterval(0, 0)} + } + var invstart int64 + ult := ni - 1 + for i, cur := range rc.iv { + if i == ult { + // invertlastInteval will add both intervals (b) and (c) in + // diagram below. + m = append(m, rc.invertlastInterval(uint16(invstart), i)...) + break + } + // INVAR: i and cur are not the last interval, there is a next at i+1 + // + // ........[cur.start, cur.last] ...... [next.start, next.last].... + // ^ ^ ^ + // (a) (b) (c) + // + // Now: we add interval (a); but if (a) is empty, for cur.start==0, we skip it. + if cur.start > 0 { + m = append(m, newInterval16Range(uint16(invstart), cur.start-1)) + } + invstart = int64(cur.last() + 1) + } + return &runContainer16{iv: m} +} + +func (iv interval16) equal(b interval16) bool { + return iv.start == b.start && iv.length == b.length +} + +func (iv interval16) isSuperSetOf(b interval16) bool { + return iv.start <= b.start && b.last() <= iv.last() +} + +func (iv interval16) subtractInterval(del interval16) (left []interval16, delcount int64) { + isect, isEmpty := intersectInterval16s(iv, del) + + if isEmpty { + return nil, 0 + } + if del.isSuperSetOf(iv) { + return nil, iv.runlen() + } + + switch { + case isect.start > iv.start && isect.last() < iv.last(): + new0 := newInterval16Range(iv.start, isect.start-1) + new1 := newInterval16Range(isect.last()+1, iv.last()) + return []interval16{new0, new1}, isect.runlen() + case isect.start == iv.start: + return []interval16{newInterval16Range(isect.last()+1, iv.last())}, isect.runlen() + default: + return []interval16{newInterval16Range(iv.start, isect.start-1)}, isect.runlen() + } +} + +func (rc *runContainer16) isubtract(del interval16) { + origiv := make([]interval16, len(rc.iv)) + copy(origiv, rc.iv) + n := int64(len(rc.iv)) + if n == 0 { + return // already done. + } + + _, isEmpty := intersectInterval16s(newInterval16Range(rc.iv[0].start, rc.iv[n-1].last()), del) + if isEmpty { + return // done + } + + // INVAR there is some intersection between rc and del + istart, startAlready, _ := rc.search(int64(del.start), nil) + ilast, lastAlready, _ := rc.search(int64(del.last()), nil) + rc.card = -1 + if istart == -1 { + if ilast == n-1 && !lastAlready { + rc.iv = nil + return + } + } + // some intervals will remain + switch { + case startAlready && lastAlready: + res0, _ := rc.iv[istart].subtractInterval(del) + + // would overwrite values in iv b/c res0 can have len 2. so + // write to origiv instead. + lost := 1 + ilast - istart + changeSize := int64(len(res0)) - lost + newSize := int64(len(rc.iv)) + changeSize + + // rc.iv = append(pre, caboose...) + // return + + if ilast != istart { + res1, _ := rc.iv[ilast].subtractInterval(del) + res0 = append(res0, res1...) + changeSize = int64(len(res0)) - lost + newSize = int64(len(rc.iv)) + changeSize + } + switch { + case changeSize < 0: + // shrink + copy(rc.iv[istart+int64(len(res0)):], rc.iv[ilast+1:]) + copy(rc.iv[istart:istart+int64(len(res0))], res0) + rc.iv = rc.iv[:newSize] + return + case changeSize == 0: + // stay the same + copy(rc.iv[istart:istart+int64(len(res0))], res0) + return + default: + // changeSize > 0 is only possible when ilast == istart. + // Hence we now know: changeSize == 1 and len(res0) == 2 + rc.iv = append(rc.iv, interval16{}) + // len(rc.iv) is correct now, no need to rc.iv = rc.iv[:newSize] + + // copy the tail into place + copy(rc.iv[ilast+2:], rc.iv[ilast+1:]) + // copy the new item(s) into place + copy(rc.iv[istart:istart+2], res0) + return + } + + case !startAlready && !lastAlready: + // we get to discard whole intervals + + // from the search() definition: + + // if del.start is not present, then istart is + // set as follows: + // + // a) istart == n-1 if del.start is beyond our + // last interval16 in rc.iv; + // + // b) istart == -1 if del.start is before our first + // interval16 in rc.iv; + // + // c) istart is set to the minimum index of rc.iv + // which comes strictly before the del.start; + // so del.start > rc.iv[istart].last, + // and if istart+1 exists, then del.start < rc.iv[istart+1].startx + + // if del.last is not present, then ilast is + // set as follows: + // + // a) ilast == n-1 if del.last is beyond our + // last interval16 in rc.iv; + // + // b) ilast == -1 if del.last is before our first + // interval16 in rc.iv; + // + // c) ilast is set to the minimum index of rc.iv + // which comes strictly before the del.last; + // so del.last > rc.iv[ilast].last, + // and if ilast+1 exists, then del.last < rc.iv[ilast+1].start + + // INVAR: istart >= 0 + pre := rc.iv[:istart+1] + if ilast == n-1 { + rc.iv = pre + return + } + // INVAR: ilast < n-1 + lost := ilast - istart + changeSize := -lost + newSize := int64(len(rc.iv)) + changeSize + if changeSize != 0 { + copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) + } + rc.iv = rc.iv[:newSize] + return + + case startAlready && !lastAlready: + // we can only shrink or stay the same size + // i.e. we either eliminate the whole interval, + // or just cut off the right side. + res0, _ := rc.iv[istart].subtractInterval(del) + if len(res0) > 0 { + // len(res) must be 1 + rc.iv[istart] = res0[0] + } + lost := 1 + (ilast - istart) + changeSize := int64(len(res0)) - lost + newSize := int64(len(rc.iv)) + changeSize + if changeSize != 0 { + copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) + } + rc.iv = rc.iv[:newSize] + return + + case !startAlready && lastAlready: + // we can only shrink or stay the same size + res1, _ := rc.iv[ilast].subtractInterval(del) + lost := ilast - istart + changeSize := int64(len(res1)) - lost + newSize := int64(len(rc.iv)) + changeSize + if changeSize != 0 { + // move the tail first to make room for res1 + copy(rc.iv[ilast+1+changeSize:], rc.iv[ilast+1:]) + } + copy(rc.iv[istart+1:], res1) + rc.iv = rc.iv[:newSize] + return + } +} + +// compute rc minus b, and return the result as a new value (not inplace). +// port of run_container_andnot from CRoaring... +// https://github.com/RoaringBitmap/CRoaring/blob/master/src/containers/run.c#L435-L496 +func (rc *runContainer16) AndNotRunContainer16(b *runContainer16) *runContainer16 { + + if len(b.iv) == 0 || len(rc.iv) == 0 { + return rc + } + + dst := newRunContainer16() + apos := 0 + bpos := 0 + + a := rc + + astart := a.iv[apos].start + alast := a.iv[apos].last() + bstart := b.iv[bpos].start + blast := b.iv[bpos].last() + + alen := len(a.iv) + blen := len(b.iv) + + for apos < alen && bpos < blen { + switch { + case alast < bstart: + // output the first run + dst.iv = append(dst.iv, newInterval16Range(astart, alast)) + apos++ + if apos < alen { + astart = a.iv[apos].start + alast = a.iv[apos].last() + } + case blast < astart: + // exit the second run + bpos++ + if bpos < blen { + bstart = b.iv[bpos].start + blast = b.iv[bpos].last() + } + default: + // a: [ ] + // b: [ ] + // alast >= bstart + // blast >= astart + if astart < bstart { + dst.iv = append(dst.iv, newInterval16Range(astart, bstart-1)) + } + if alast > blast { + astart = blast + 1 + } else { + apos++ + if apos < alen { + astart = a.iv[apos].start + alast = a.iv[apos].last() + } + } + } + } + if apos < alen { + dst.iv = append(dst.iv, newInterval16Range(astart, alast)) + apos++ + if apos < alen { + dst.iv = append(dst.iv, a.iv[apos:]...) + } + } + + return dst +} + +func (rc *runContainer16) numberOfRuns() (nr int) { + return len(rc.iv) +} + +func (rc *runContainer16) containerType() contype { + return run16Contype +} + +func (rc *runContainer16) equals16(srb *runContainer16) bool { + //p("both rc16") + // Check if the containers are the same object. + if rc == srb { + //p("same object") + return true + } + + if len(srb.iv) != len(rc.iv) { + //p("iv len differ") + return false + } + + for i, v := range rc.iv { + if v != srb.iv[i] { + //p("differ at iv i=%v, srb.iv[i]=%v, rc.iv[i]=%v", i, srb.iv[i], rc.iv[i]) + return false + } + } + //p("all intervals same, returning true") + return true +} diff --git a/vendor/github.com/RoaringBitmap/roaring/rle16_gen.go b/vendor/github.com/RoaringBitmap/roaring/rle16_gen.go new file mode 100644 index 00000000..05bf4463 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/rle16_gen.go @@ -0,0 +1,1126 @@ +package roaring + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import "github.com/tinylib/msgp/msgp" + +// DecodeMsg implements msgp.Decodable +func (z *addHelper16) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zbai uint32 + zbai, err = dc.ReadMapHeader() + if err != nil { + return + } + for zbai > 0 { + zbai-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "runstart": + z.runstart, err = dc.ReadUint16() + if err != nil { + return + } + case "runlen": + z.runlen, err = dc.ReadUint16() + if err != nil { + return + } + case "actuallyAdded": + z.actuallyAdded, err = dc.ReadUint16() + if err != nil { + return + } + case "m": + var zcmr uint32 + zcmr, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.m) >= int(zcmr) { + z.m = (z.m)[:zcmr] + } else { + z.m = make([]interval16, zcmr) + } + for zxvk := range z.m { + var zajw uint32 + zajw, err = dc.ReadMapHeader() + if err != nil { + return + } + for zajw > 0 { + zajw-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.m[zxvk].start, err = dc.ReadUint16() + if err != nil { + return + } + case "last": + z.m[zxvk].length, err = dc.ReadUint16() + z.m[zxvk].length -= z.m[zxvk].start + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + case "rc": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + return + } + z.rc = nil + } else { + if z.rc == nil { + z.rc = new(runContainer16) + } + var zwht uint32 + zwht, err = dc.ReadMapHeader() + if err != nil { + return + } + for zwht > 0 { + zwht-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "iv": + var zhct uint32 + zhct, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.rc.iv) >= int(zhct) { + z.rc.iv = (z.rc.iv)[:zhct] + } else { + z.rc.iv = make([]interval16, zhct) + } + for zbzg := range z.rc.iv { + var zcua uint32 + zcua, err = dc.ReadMapHeader() + if err != nil { + return + } + for zcua > 0 { + zcua-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.rc.iv[zbzg].start, err = dc.ReadUint16() + if err != nil { + return + } + case "last": + z.rc.iv[zbzg].length, err = dc.ReadUint16() + z.rc.iv[zbzg].length -= z.rc.iv[zbzg].start + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + case "card": + z.rc.card, err = dc.ReadInt64() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *addHelper16) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 5 + // write "runstart" + err = en.Append(0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.runstart) + if err != nil { + return + } + // write "runlen" + err = en.Append(0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e) + if err != nil { + return err + } + err = en.WriteUint16(z.runlen) + if err != nil { + return + } + // write "actuallyAdded" + err = en.Append(0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64) + if err != nil { + return err + } + err = en.WriteUint16(z.actuallyAdded) + if err != nil { + return + } + // write "m" + err = en.Append(0xa1, 0x6d) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.m))) + if err != nil { + return + } + for zxvk := range z.m { + // map header, size 2 + // write "start" + err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.m[zxvk].start) + if err != nil { + return + } + // write "last" + err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.m[zxvk].last()) + if err != nil { + return + } + } + // write "rc" + err = en.Append(0xa2, 0x72, 0x63) + if err != nil { + return err + } + if z.rc == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + // map header, size 2 + // write "iv" + err = en.Append(0x82, 0xa2, 0x69, 0x76) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.rc.iv))) + if err != nil { + return + } + for zbzg := range z.rc.iv { + // map header, size 2 + // write "start" + err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.rc.iv[zbzg].start) + if err != nil { + return + } + // write "last" + err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.rc.iv[zbzg].last()) + if err != nil { + return + } + } + // write "card" + err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64) + if err != nil { + return err + } + err = en.WriteInt64(z.rc.card) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *addHelper16) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "runstart" + o = append(o, 0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint16(o, z.runstart) + // string "runlen" + o = append(o, 0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e) + o = msgp.AppendUint16(o, z.runlen) + // string "actuallyAdded" + o = append(o, 0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64) + o = msgp.AppendUint16(o, z.actuallyAdded) + // string "m" + o = append(o, 0xa1, 0x6d) + o = msgp.AppendArrayHeader(o, uint32(len(z.m))) + for zxvk := range z.m { + // map header, size 2 + // string "start" + o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint16(o, z.m[zxvk].start) + // string "last" + o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) + o = msgp.AppendUint16(o, z.m[zxvk].last()) + } + // string "rc" + o = append(o, 0xa2, 0x72, 0x63) + if z.rc == nil { + o = msgp.AppendNil(o) + } else { + // map header, size 2 + // string "iv" + o = append(o, 0x82, 0xa2, 0x69, 0x76) + o = msgp.AppendArrayHeader(o, uint32(len(z.rc.iv))) + for zbzg := range z.rc.iv { + // map header, size 2 + // string "start" + o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint16(o, z.rc.iv[zbzg].start) + // string "last" + o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) + o = msgp.AppendUint16(o, z.rc.iv[zbzg].last()) + } + // string "card" + o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64) + o = msgp.AppendInt64(o, z.rc.card) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *addHelper16) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zxhx uint32 + zxhx, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zxhx > 0 { + zxhx-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "runstart": + z.runstart, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "runlen": + z.runlen, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "actuallyAdded": + z.actuallyAdded, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "m": + var zlqf uint32 + zlqf, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.m) >= int(zlqf) { + z.m = (z.m)[:zlqf] + } else { + z.m = make([]interval16, zlqf) + } + for zxvk := range z.m { + var zdaf uint32 + zdaf, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zdaf > 0 { + zdaf-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.m[zxvk].start, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "last": + z.m[zxvk].length, bts, err = msgp.ReadUint16Bytes(bts) + z.m[zxvk].length -= z.m[zxvk].start + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + case "rc": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.rc = nil + } else { + if z.rc == nil { + z.rc = new(runContainer16) + } + var zpks uint32 + zpks, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zpks > 0 { + zpks-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "iv": + var zjfb uint32 + zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.rc.iv) >= int(zjfb) { + z.rc.iv = (z.rc.iv)[:zjfb] + } else { + z.rc.iv = make([]interval16, zjfb) + } + for zbzg := range z.rc.iv { + var zcxo uint32 + zcxo, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zcxo > 0 { + zcxo-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.rc.iv[zbzg].start, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "last": + z.rc.iv[zbzg].length, bts, err = msgp.ReadUint16Bytes(bts) + z.rc.iv[zbzg].length -= z.rc.iv[zbzg].start + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + case "card": + z.rc.card, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *addHelper16) Msgsize() (s int) { + s = 1 + 9 + msgp.Uint16Size + 7 + msgp.Uint16Size + 14 + msgp.Uint16Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 3 + if z.rc == nil { + s += msgp.NilSize + } else { + s += 1 + 3 + msgp.ArrayHeaderSize + (len(z.rc.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *interval16) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zeff uint32 + zeff, err = dc.ReadMapHeader() + if err != nil { + return + } + for zeff > 0 { + zeff-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.start, err = dc.ReadUint16() + if err != nil { + return + } + case "last": + z.length, err = dc.ReadUint16() + z.length = -z.start + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z interval16) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "start" + err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.start) + if err != nil { + return + } + // write "last" + err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.last()) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z interval16) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "start" + o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint16(o, z.start) + // string "last" + o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) + o = msgp.AppendUint16(o, z.last()) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *interval16) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zrsw uint32 + zrsw, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zrsw > 0 { + zrsw-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.start, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "last": + z.length, bts, err = msgp.ReadUint16Bytes(bts) + z.length -= z.start + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z interval16) Msgsize() (s int) { + s = 1 + 6 + msgp.Uint16Size + 5 + msgp.Uint16Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *runContainer16) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zdnj uint32 + zdnj, err = dc.ReadMapHeader() + if err != nil { + return + } + for zdnj > 0 { + zdnj-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "iv": + var zobc uint32 + zobc, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.iv) >= int(zobc) { + z.iv = (z.iv)[:zobc] + } else { + z.iv = make([]interval16, zobc) + } + for zxpk := range z.iv { + var zsnv uint32 + zsnv, err = dc.ReadMapHeader() + if err != nil { + return + } + for zsnv > 0 { + zsnv-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.iv[zxpk].start, err = dc.ReadUint16() + if err != nil { + return + } + case "last": + z.iv[zxpk].length, err = dc.ReadUint16() + z.iv[zxpk].length -= z.iv[zxpk].start + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + case "card": + z.card, err = dc.ReadInt64() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *runContainer16) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "iv" + err = en.Append(0x82, 0xa2, 0x69, 0x76) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.iv))) + if err != nil { + return + } + for zxpk := range z.iv { + // map header, size 2 + // write "start" + err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.iv[zxpk].start) + if err != nil { + return + } + // write "last" + err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) + if err != nil { + return err + } + err = en.WriteUint16(z.iv[zxpk].last()) + if err != nil { + return + } + } + // write "card" + err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64) + if err != nil { + return err + } + err = en.WriteInt64(z.card) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *runContainer16) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "iv" + o = append(o, 0x82, 0xa2, 0x69, 0x76) + o = msgp.AppendArrayHeader(o, uint32(len(z.iv))) + for zxpk := range z.iv { + // map header, size 2 + // string "start" + o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint16(o, z.iv[zxpk].start) + // string "last" + o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) + o = msgp.AppendUint16(o, z.iv[zxpk].last()) + } + // string "card" + o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64) + o = msgp.AppendInt64(o, z.card) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *runContainer16) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zkgt uint32 + zkgt, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zkgt > 0 { + zkgt-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "iv": + var zema uint32 + zema, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.iv) >= int(zema) { + z.iv = (z.iv)[:zema] + } else { + z.iv = make([]interval16, zema) + } + for zxpk := range z.iv { + var zpez uint32 + zpez, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zpez > 0 { + zpez-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.iv[zxpk].start, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "last": + z.iv[zxpk].length, bts, err = msgp.ReadUint16Bytes(bts) + z.iv[zxpk].length -= z.iv[zxpk].start + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + case "card": + z.card, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *runContainer16) Msgsize() (s int) { + s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint16Size + msgp.Uint16Size)) + 5 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *runIterator16) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zqke uint32 + zqke, err = dc.ReadMapHeader() + if err != nil { + return + } + for zqke > 0 { + zqke-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "rc": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + return + } + z.rc = nil + } else { + if z.rc == nil { + z.rc = new(runContainer16) + } + err = z.rc.DecodeMsg(dc) + if err != nil { + return + } + } + case "curIndex": + z.curIndex, err = dc.ReadInt64() + if err != nil { + return + } + case "curPosInIndex": + z.curPosInIndex, err = dc.ReadUint16() + if err != nil { + return + } + case "curSeq": + z.curSeq, err = dc.ReadInt64() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *runIterator16) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "rc" + err = en.Append(0x84, 0xa2, 0x72, 0x63) + if err != nil { + return err + } + if z.rc == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.rc.EncodeMsg(en) + if err != nil { + return + } + } + // write "curIndex" + err = en.Append(0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return err + } + err = en.WriteInt64(z.curIndex) + if err != nil { + return + } + // write "curPosInIndex" + err = en.Append(0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return err + } + err = en.WriteUint16(z.curPosInIndex) + if err != nil { + return + } + // write "curSeq" + err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71) + if err != nil { + return err + } + err = en.WriteInt64(z.curSeq) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *runIterator16) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "rc" + o = append(o, 0x84, 0xa2, 0x72, 0x63) + if z.rc == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.rc.MarshalMsg(o) + if err != nil { + return + } + } + // string "curIndex" + o = append(o, 0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt64(o, z.curIndex) + // string "curPosInIndex" + o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendUint16(o, z.curPosInIndex) + // string "curSeq" + o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71) + o = msgp.AppendInt64(o, z.curSeq) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *runIterator16) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zqyh uint32 + zqyh, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zqyh > 0 { + zqyh-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "rc": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.rc = nil + } else { + if z.rc == nil { + z.rc = new(runContainer16) + } + bts, err = z.rc.UnmarshalMsg(bts) + if err != nil { + return + } + } + case "curIndex": + z.curIndex, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + return + } + case "curPosInIndex": + z.curPosInIndex, bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + case "curSeq": + z.curSeq, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *runIterator16) Msgsize() (s int) { + s = 1 + 3 + if z.rc == nil { + s += msgp.NilSize + } else { + s += z.rc.Msgsize() + } + s += 9 + msgp.Int64Size + 14 + msgp.Uint16Size + 7 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *uint16Slice) DecodeMsg(dc *msgp.Reader) (err error) { + var zjpj uint32 + zjpj, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap((*z)) >= int(zjpj) { + (*z) = (*z)[:zjpj] + } else { + (*z) = make(uint16Slice, zjpj) + } + for zywj := range *z { + (*z)[zywj], err = dc.ReadUint16() + if err != nil { + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z uint16Slice) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(len(z))) + if err != nil { + return + } + for zzpf := range z { + err = en.WriteUint16(z[zzpf]) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z uint16Slice) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(len(z))) + for zzpf := range z { + o = msgp.AppendUint16(o, z[zzpf]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *uint16Slice) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zgmo uint32 + zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap((*z)) >= int(zgmo) { + (*z) = (*z)[:zgmo] + } else { + (*z) = make(uint16Slice, zgmo) + } + for zrfe := range *z { + (*z)[zrfe], bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z uint16Slice) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint16Size)) + return +} diff --git a/vendor/github.com/RoaringBitmap/roaring/rle_gen.go b/vendor/github.com/RoaringBitmap/roaring/rle_gen.go new file mode 100644 index 00000000..bc9da75f --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/rle_gen.go @@ -0,0 +1,1118 @@ +package roaring + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import "github.com/tinylib/msgp/msgp" + +// DecodeMsg implements msgp.Decodable +func (z *addHelper32) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zbai uint32 + zbai, err = dc.ReadMapHeader() + if err != nil { + return + } + for zbai > 0 { + zbai-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "runstart": + z.runstart, err = dc.ReadUint32() + if err != nil { + return + } + case "runlen": + z.runlen, err = dc.ReadUint32() + if err != nil { + return + } + case "actuallyAdded": + z.actuallyAdded, err = dc.ReadUint32() + if err != nil { + return + } + case "m": + var zcmr uint32 + zcmr, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.m) >= int(zcmr) { + z.m = (z.m)[:zcmr] + } else { + z.m = make([]interval32, zcmr) + } + for zxvk := range z.m { + var zajw uint32 + zajw, err = dc.ReadMapHeader() + if err != nil { + return + } + for zajw > 0 { + zajw-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.m[zxvk].start, err = dc.ReadUint32() + if err != nil { + return + } + case "last": + z.m[zxvk].last, err = dc.ReadUint32() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + case "rc": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + return + } + z.rc = nil + } else { + if z.rc == nil { + z.rc = new(runContainer32) + } + var zwht uint32 + zwht, err = dc.ReadMapHeader() + if err != nil { + return + } + for zwht > 0 { + zwht-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "iv": + var zhct uint32 + zhct, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.rc.iv) >= int(zhct) { + z.rc.iv = (z.rc.iv)[:zhct] + } else { + z.rc.iv = make([]interval32, zhct) + } + for zbzg := range z.rc.iv { + var zcua uint32 + zcua, err = dc.ReadMapHeader() + if err != nil { + return + } + for zcua > 0 { + zcua-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.rc.iv[zbzg].start, err = dc.ReadUint32() + if err != nil { + return + } + case "last": + z.rc.iv[zbzg].last, err = dc.ReadUint32() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + case "card": + z.rc.card, err = dc.ReadInt64() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *addHelper32) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 5 + // write "runstart" + err = en.Append(0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.runstart) + if err != nil { + return + } + // write "runlen" + err = en.Append(0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e) + if err != nil { + return err + } + err = en.WriteUint32(z.runlen) + if err != nil { + return + } + // write "actuallyAdded" + err = en.Append(0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64) + if err != nil { + return err + } + err = en.WriteUint32(z.actuallyAdded) + if err != nil { + return + } + // write "m" + err = en.Append(0xa1, 0x6d) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.m))) + if err != nil { + return + } + for zxvk := range z.m { + // map header, size 2 + // write "start" + err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.m[zxvk].start) + if err != nil { + return + } + // write "last" + err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.m[zxvk].last) + if err != nil { + return + } + } + // write "rc" + err = en.Append(0xa2, 0x72, 0x63) + if err != nil { + return err + } + if z.rc == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + // map header, size 2 + // write "iv" + err = en.Append(0x82, 0xa2, 0x69, 0x76) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.rc.iv))) + if err != nil { + return + } + for zbzg := range z.rc.iv { + // map header, size 2 + // write "start" + err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.rc.iv[zbzg].start) + if err != nil { + return + } + // write "last" + err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.rc.iv[zbzg].last) + if err != nil { + return + } + } + // write "card" + err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64) + if err != nil { + return err + } + err = en.WriteInt64(z.rc.card) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *addHelper32) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "runstart" + o = append(o, 0x85, 0xa8, 0x72, 0x75, 0x6e, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint32(o, z.runstart) + // string "runlen" + o = append(o, 0xa6, 0x72, 0x75, 0x6e, 0x6c, 0x65, 0x6e) + o = msgp.AppendUint32(o, z.runlen) + // string "actuallyAdded" + o = append(o, 0xad, 0x61, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x6c, 0x79, 0x41, 0x64, 0x64, 0x65, 0x64) + o = msgp.AppendUint32(o, z.actuallyAdded) + // string "m" + o = append(o, 0xa1, 0x6d) + o = msgp.AppendArrayHeader(o, uint32(len(z.m))) + for zxvk := range z.m { + // map header, size 2 + // string "start" + o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint32(o, z.m[zxvk].start) + // string "last" + o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) + o = msgp.AppendUint32(o, z.m[zxvk].last) + } + // string "rc" + o = append(o, 0xa2, 0x72, 0x63) + if z.rc == nil { + o = msgp.AppendNil(o) + } else { + // map header, size 2 + // string "iv" + o = append(o, 0x82, 0xa2, 0x69, 0x76) + o = msgp.AppendArrayHeader(o, uint32(len(z.rc.iv))) + for zbzg := range z.rc.iv { + // map header, size 2 + // string "start" + o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint32(o, z.rc.iv[zbzg].start) + // string "last" + o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) + o = msgp.AppendUint32(o, z.rc.iv[zbzg].last) + } + // string "card" + o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64) + o = msgp.AppendInt64(o, z.rc.card) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *addHelper32) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zxhx uint32 + zxhx, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zxhx > 0 { + zxhx-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "runstart": + z.runstart, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "runlen": + z.runlen, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "actuallyAdded": + z.actuallyAdded, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "m": + var zlqf uint32 + zlqf, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.m) >= int(zlqf) { + z.m = (z.m)[:zlqf] + } else { + z.m = make([]interval32, zlqf) + } + for zxvk := range z.m { + var zdaf uint32 + zdaf, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zdaf > 0 { + zdaf-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.m[zxvk].start, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "last": + z.m[zxvk].last, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + case "rc": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.rc = nil + } else { + if z.rc == nil { + z.rc = new(runContainer32) + } + var zpks uint32 + zpks, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zpks > 0 { + zpks-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "iv": + var zjfb uint32 + zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.rc.iv) >= int(zjfb) { + z.rc.iv = (z.rc.iv)[:zjfb] + } else { + z.rc.iv = make([]interval32, zjfb) + } + for zbzg := range z.rc.iv { + var zcxo uint32 + zcxo, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zcxo > 0 { + zcxo-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.rc.iv[zbzg].start, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "last": + z.rc.iv[zbzg].last, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + case "card": + z.rc.card, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *addHelper32) Msgsize() (s int) { + s = 1 + 9 + msgp.Uint32Size + 7 + msgp.Uint32Size + 14 + msgp.Uint32Size + 2 + msgp.ArrayHeaderSize + (len(z.m) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 3 + if z.rc == nil { + s += msgp.NilSize + } else { + s += 1 + 3 + msgp.ArrayHeaderSize + (len(z.rc.iv) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 5 + msgp.Int64Size + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *interval32) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zeff uint32 + zeff, err = dc.ReadMapHeader() + if err != nil { + return + } + for zeff > 0 { + zeff-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.start, err = dc.ReadUint32() + if err != nil { + return + } + case "last": + z.last, err = dc.ReadUint32() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z interval32) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "start" + err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.start) + if err != nil { + return + } + // write "last" + err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.last) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z interval32) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "start" + o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint32(o, z.start) + // string "last" + o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) + o = msgp.AppendUint32(o, z.last) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *interval32) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zrsw uint32 + zrsw, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zrsw > 0 { + zrsw-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.start, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "last": + z.last, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z interval32) Msgsize() (s int) { + s = 1 + 6 + msgp.Uint32Size + 5 + msgp.Uint32Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *runContainer32) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zdnj uint32 + zdnj, err = dc.ReadMapHeader() + if err != nil { + return + } + for zdnj > 0 { + zdnj-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "iv": + var zobc uint32 + zobc, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.iv) >= int(zobc) { + z.iv = (z.iv)[:zobc] + } else { + z.iv = make([]interval32, zobc) + } + for zxpk := range z.iv { + var zsnv uint32 + zsnv, err = dc.ReadMapHeader() + if err != nil { + return + } + for zsnv > 0 { + zsnv-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.iv[zxpk].start, err = dc.ReadUint32() + if err != nil { + return + } + case "last": + z.iv[zxpk].last, err = dc.ReadUint32() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + case "card": + z.card, err = dc.ReadInt64() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *runContainer32) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "iv" + err = en.Append(0x82, 0xa2, 0x69, 0x76) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.iv))) + if err != nil { + return + } + for zxpk := range z.iv { + // map header, size 2 + // write "start" + err = en.Append(0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.iv[zxpk].start) + if err != nil { + return + } + // write "last" + err = en.Append(0xa4, 0x6c, 0x61, 0x73, 0x74) + if err != nil { + return err + } + err = en.WriteUint32(z.iv[zxpk].last) + if err != nil { + return + } + } + // write "card" + err = en.Append(0xa4, 0x63, 0x61, 0x72, 0x64) + if err != nil { + return err + } + err = en.WriteInt64(z.card) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *runContainer32) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "iv" + o = append(o, 0x82, 0xa2, 0x69, 0x76) + o = msgp.AppendArrayHeader(o, uint32(len(z.iv))) + for zxpk := range z.iv { + // map header, size 2 + // string "start" + o = append(o, 0x82, 0xa5, 0x73, 0x74, 0x61, 0x72, 0x74) + o = msgp.AppendUint32(o, z.iv[zxpk].start) + // string "last" + o = append(o, 0xa4, 0x6c, 0x61, 0x73, 0x74) + o = msgp.AppendUint32(o, z.iv[zxpk].last) + } + // string "card" + o = append(o, 0xa4, 0x63, 0x61, 0x72, 0x64) + o = msgp.AppendInt64(o, z.card) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *runContainer32) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zkgt uint32 + zkgt, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zkgt > 0 { + zkgt-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "iv": + var zema uint32 + zema, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.iv) >= int(zema) { + z.iv = (z.iv)[:zema] + } else { + z.iv = make([]interval32, zema) + } + for zxpk := range z.iv { + var zpez uint32 + zpez, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zpez > 0 { + zpez-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "start": + z.iv[zxpk].start, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "last": + z.iv[zxpk].last, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + case "card": + z.card, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *runContainer32) Msgsize() (s int) { + s = 1 + 3 + msgp.ArrayHeaderSize + (len(z.iv) * (12 + msgp.Uint32Size + msgp.Uint32Size)) + 5 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *runIterator32) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zqke uint32 + zqke, err = dc.ReadMapHeader() + if err != nil { + return + } + for zqke > 0 { + zqke-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "rc": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + return + } + z.rc = nil + } else { + if z.rc == nil { + z.rc = new(runContainer32) + } + err = z.rc.DecodeMsg(dc) + if err != nil { + return + } + } + case "curIndex": + z.curIndex, err = dc.ReadInt64() + if err != nil { + return + } + case "curPosInIndex": + z.curPosInIndex, err = dc.ReadUint32() + if err != nil { + return + } + case "curSeq": + z.curSeq, err = dc.ReadInt64() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *runIterator32) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "rc" + err = en.Append(0x84, 0xa2, 0x72, 0x63) + if err != nil { + return err + } + if z.rc == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.rc.EncodeMsg(en) + if err != nil { + return + } + } + // write "curIndex" + err = en.Append(0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return err + } + err = en.WriteInt64(z.curIndex) + if err != nil { + return + } + // write "curPosInIndex" + err = en.Append(0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return err + } + err = en.WriteUint32(z.curPosInIndex) + if err != nil { + return + } + // write "curSeq" + err = en.Append(0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71) + if err != nil { + return err + } + err = en.WriteInt64(z.curSeq) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *runIterator32) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "rc" + o = append(o, 0x84, 0xa2, 0x72, 0x63) + if z.rc == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.rc.MarshalMsg(o) + if err != nil { + return + } + } + // string "curIndex" + o = append(o, 0xa8, 0x63, 0x75, 0x72, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt64(o, z.curIndex) + // string "curPosInIndex" + o = append(o, 0xad, 0x63, 0x75, 0x72, 0x50, 0x6f, 0x73, 0x49, 0x6e, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendUint32(o, z.curPosInIndex) + // string "curSeq" + o = append(o, 0xa6, 0x63, 0x75, 0x72, 0x53, 0x65, 0x71) + o = msgp.AppendInt64(o, z.curSeq) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *runIterator32) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zqyh uint32 + zqyh, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zqyh > 0 { + zqyh-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "rc": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.rc = nil + } else { + if z.rc == nil { + z.rc = new(runContainer32) + } + bts, err = z.rc.UnmarshalMsg(bts) + if err != nil { + return + } + } + case "curIndex": + z.curIndex, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + return + } + case "curPosInIndex": + z.curPosInIndex, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + case "curSeq": + z.curSeq, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *runIterator32) Msgsize() (s int) { + s = 1 + 3 + if z.rc == nil { + s += msgp.NilSize + } else { + s += z.rc.Msgsize() + } + s += 9 + msgp.Int64Size + 14 + msgp.Uint32Size + 7 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *uint32Slice) DecodeMsg(dc *msgp.Reader) (err error) { + var zjpj uint32 + zjpj, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap((*z)) >= int(zjpj) { + (*z) = (*z)[:zjpj] + } else { + (*z) = make(uint32Slice, zjpj) + } + for zywj := range *z { + (*z)[zywj], err = dc.ReadUint32() + if err != nil { + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z uint32Slice) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(len(z))) + if err != nil { + return + } + for zzpf := range z { + err = en.WriteUint32(z[zzpf]) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z uint32Slice) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(len(z))) + for zzpf := range z { + o = msgp.AppendUint32(o, z[zzpf]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *uint32Slice) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zgmo uint32 + zgmo, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap((*z)) >= int(zgmo) { + (*z) = (*z)[:zgmo] + } else { + (*z) = make(uint32Slice, zgmo) + } + for zrfe := range *z { + (*z)[zrfe], bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z uint32Slice) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + (len(z) * (msgp.Uint32Size)) + return +} diff --git a/vendor/github.com/RoaringBitmap/roaring/rlecommon.go b/vendor/github.com/RoaringBitmap/roaring/rlecommon.go new file mode 100644 index 00000000..13363678 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/rlecommon.go @@ -0,0 +1,163 @@ +package roaring + +import ( + "fmt" +) + +// common to rle32.go and rle16.go + +// rleVerbose controls whether p() prints show up. +// The testing package sets this based on +// testing.Verbose(). +var rleVerbose bool + +// p is a shorthand for fmt.Printf with beginning and +// trailing newlines. p() makes it easy +// to add diagnostic print statements. +func p(format string, args ...interface{}) { + if rleVerbose { + fmt.Printf("\n"+format+"\n", args...) + } +} + +// MaxUint32 is the largest uint32 value. +const MaxUint32 = 4294967295 + +// MaxUint16 is the largest 16 bit unsigned int. +// This is the largest value an interval16 can store. +const MaxUint16 = 65535 + +// searchOptions allows us to accelerate runContainer32.search with +// prior knowledge of (mostly lower) bounds. This is used by Union +// and Intersect. +type searchOptions struct { + // start here instead of at 0 + startIndex int64 + + // upper bound instead of len(rc.iv); + // endxIndex == 0 means ignore the bound and use + // endxIndex == n ==len(rc.iv) which is also + // naturally the default for search() + // when opt = nil. + endxIndex int64 +} + +// And finds the intersection of rc and b. +func (rc *runContainer32) And(b *Bitmap) *Bitmap { + out := NewBitmap() + for _, p := range rc.iv { + for i := p.start; i <= p.last; i++ { + if b.Contains(i) { + out.Add(i) + } + } + } + return out +} + +// Xor returns the exclusive-or of rc and b. +func (rc *runContainer32) Xor(b *Bitmap) *Bitmap { + out := b.Clone() + for _, p := range rc.iv { + for v := p.start; v <= p.last; v++ { + if out.Contains(v) { + out.RemoveRange(uint64(v), uint64(v+1)) + } else { + out.Add(v) + } + } + } + return out +} + +// Or returns the union of rc and b. +func (rc *runContainer32) Or(b *Bitmap) *Bitmap { + out := b.Clone() + for _, p := range rc.iv { + for v := p.start; v <= p.last; v++ { + out.Add(v) + } + } + return out +} + +// trial is used in the randomized testing of runContainers +type trial struct { + n int + percentFill float64 + ntrial int + + // only in the union test + // only subtract test + percentDelete float64 + + // only in 067 randomized operations + // we do this + 1 passes + numRandomOpsPass int + + // allow sampling range control + // only recent tests respect this. + srang *interval16 +} + +// And finds the intersection of rc and b. +func (rc *runContainer16) And(b *Bitmap) *Bitmap { + out := NewBitmap() + for _, p := range rc.iv { + plast := p.last() + for i := p.start; i <= plast; i++ { + if b.Contains(uint32(i)) { + out.Add(uint32(i)) + } + } + } + return out +} + +// Xor returns the exclusive-or of rc and b. +func (rc *runContainer16) Xor(b *Bitmap) *Bitmap { + out := b.Clone() + for _, p := range rc.iv { + plast := p.last() + for v := p.start; v <= plast; v++ { + w := uint32(v) + if out.Contains(w) { + out.RemoveRange(uint64(w), uint64(w+1)) + } else { + out.Add(w) + } + } + } + return out +} + +// Or returns the union of rc and b. +func (rc *runContainer16) Or(b *Bitmap) *Bitmap { + out := b.Clone() + for _, p := range rc.iv { + plast := p.last() + for v := p.start; v <= plast; v++ { + out.Add(uint32(v)) + } + } + return out +} + +//func (rc *runContainer32) and(container) container { +// panic("TODO. not yet implemented") +//} + +// serializedSizeInBytes returns the number of bytes of memory +// required by this runContainer16. This is for the +// Roaring format, as specified https://github.com/RoaringBitmap/RoaringFormatSpec/ +func (rc *runContainer16) serializedSizeInBytes() int { + // number of runs in one uint16, then each run + // needs two more uint16 + return 2 + len(rc.iv)*4 +} + +// serializedSizeInBytes returns the number of bytes of memory +// required by this runContainer32. +func (rc *runContainer32) serializedSizeInBytes() int { + return 4 + len(rc.iv)*8 +} diff --git a/vendor/github.com/RoaringBitmap/roaring/rlei.go b/vendor/github.com/RoaringBitmap/roaring/rlei.go new file mode 100644 index 00000000..a15a017e --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/rlei.go @@ -0,0 +1,695 @@ +package roaring + +/////////////////////////////////////////////////// +// +// container interface methods for runContainer16 +// +/////////////////////////////////////////////////// + +import ( + "fmt" +) + +// compile time verify we meet interface requirements +var _ container = &runContainer16{} + +func (rc *runContainer16) clone() container { + return newRunContainer16CopyIv(rc.iv) +} + +func (rc *runContainer16) minimum() uint16 { + return rc.iv[0].start // assume not empty +} + +func (rc *runContainer16) maximum() uint16 { + return rc.iv[len(rc.iv)-1].last() // assume not empty +} + +func (rc *runContainer16) isFull() bool { + return (len(rc.iv) == 1) && ((rc.iv[0].start == 0) && (rc.iv[0].last() == MaxUint16)) +} + +func (rc *runContainer16) and(a container) container { + if rc.isFull() { + return a.clone() + } + switch c := a.(type) { + case *runContainer16: + return rc.intersect(c) + case *arrayContainer: + return rc.andArray(c) + case *bitmapContainer: + return rc.andBitmapContainer(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) andCardinality(a container) int { + switch c := a.(type) { + case *runContainer16: + return int(rc.intersectCardinality(c)) + case *arrayContainer: + return rc.andArrayCardinality(c) + case *bitmapContainer: + return rc.andBitmapContainerCardinality(c) + } + panic("unsupported container type") +} + +// andBitmapContainer finds the intersection of rc and b. +func (rc *runContainer16) andBitmapContainer(bc *bitmapContainer) container { + bc2 := newBitmapContainerFromRun(rc) + return bc2.andBitmap(bc) +} + +func (rc *runContainer16) andArrayCardinality(ac *arrayContainer) int { + pos := 0 + answer := 0 + maxpos := ac.getCardinality() + if maxpos == 0 { + return 0 // won't happen in actual code + } + v := ac.content[pos] +mainloop: + for _, p := range rc.iv { + for v < p.start { + pos++ + if pos == maxpos { + break mainloop + } + v = ac.content[pos] + } + for v <= p.last() { + answer++ + pos++ + if pos == maxpos { + break mainloop + } + v = ac.content[pos] + } + } + return answer +} + +func (rc *runContainer16) iand(a container) container { + if rc.isFull() { + return a.clone() + } + switch c := a.(type) { + case *runContainer16: + return rc.inplaceIntersect(c) + case *arrayContainer: + return rc.andArray(c) + case *bitmapContainer: + return rc.iandBitmapContainer(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) inplaceIntersect(rc2 *runContainer16) container { + // TODO: optimize by doing less allocation, possibly? + + // sect will be new + sect := rc.intersect(rc2) + *rc = *sect + return rc +} + +func (rc *runContainer16) iandBitmapContainer(bc *bitmapContainer) container { + isect := rc.andBitmapContainer(bc) + *rc = *newRunContainer16FromContainer(isect) + return rc +} + +func (rc *runContainer16) andArray(ac *arrayContainer) container { + if len(rc.iv) == 0 { + return newArrayContainer() + } + + acCardinality := ac.getCardinality() + c := newArrayContainerCapacity(acCardinality) + + for rlePos, arrayPos := 0, 0; arrayPos < acCardinality; { + iv := rc.iv[rlePos] + arrayVal := ac.content[arrayPos] + + for iv.last() < arrayVal { + rlePos++ + if rlePos == len(rc.iv) { + return c + } + iv = rc.iv[rlePos] + } + + if iv.start > arrayVal { + arrayPos = advanceUntil(ac.content, arrayPos, len(ac.content), iv.start) + } else { + c.content = append(c.content, arrayVal) + arrayPos++ + } + } + return c +} + +func (rc *runContainer16) andNot(a container) container { + switch c := a.(type) { + case *arrayContainer: + return rc.andNotArray(c) + case *bitmapContainer: + return rc.andNotBitmap(c) + case *runContainer16: + return rc.andNotRunContainer16(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) fillLeastSignificant16bits(x []uint32, i int, mask uint32) { + k := 0 + var val int64 + for _, p := range rc.iv { + n := p.runlen() + for j := int64(0); j < n; j++ { + val = int64(p.start) + j + x[k+i] = uint32(val) | mask + k++ + } + } +} + +func (rc *runContainer16) getShortIterator() shortIterable { + return rc.newRunIterator16() +} + +func (rc *runContainer16) getManyIterator() manyIterable { + return rc.newManyRunIterator16() +} + +// add the values in the range [firstOfRange, endx). endx +// is still abe to express 2^16 because it is an int not an uint16. +func (rc *runContainer16) iaddRange(firstOfRange, endx int) container { + + if firstOfRange >= endx { + panic(fmt.Sprintf("invalid %v = endx >= firstOfRange", endx)) + } + addme := newRunContainer16TakeOwnership([]interval16{ + { + start: uint16(firstOfRange), + length: uint16(endx - 1 - firstOfRange), + }, + }) + *rc = *rc.union(addme) + return rc +} + +// remove the values in the range [firstOfRange,endx) +func (rc *runContainer16) iremoveRange(firstOfRange, endx int) container { + if firstOfRange >= endx { + panic(fmt.Sprintf("request to iremove empty set [%v, %v),"+ + " nothing to do.", firstOfRange, endx)) + //return rc + } + x := newInterval16Range(uint16(firstOfRange), uint16(endx-1)) + rc.isubtract(x) + return rc +} + +// not flip the values in the range [firstOfRange,endx) +func (rc *runContainer16) not(firstOfRange, endx int) container { + if firstOfRange >= endx { + panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange)) + } + + return rc.Not(firstOfRange, endx) +} + +// Not flips the values in the range [firstOfRange,endx). +// This is not inplace. Only the returned value has the flipped bits. +// +// Currently implemented as (!A intersect B) union (A minus B), +// where A is rc, and B is the supplied [firstOfRange, endx) interval. +// +// TODO(time optimization): convert this to a single pass +// algorithm by copying AndNotRunContainer16() and modifying it. +// Current routine is correct but +// makes 2 more passes through the arrays than should be +// strictly necessary. Measure both ways though--this may not matter. +// +func (rc *runContainer16) Not(firstOfRange, endx int) *runContainer16 { + + if firstOfRange >= endx { + panic(fmt.Sprintf("invalid %v = endx >= firstOfRange == %v", endx, firstOfRange)) + } + + if firstOfRange >= endx { + return rc.Clone() + } + + a := rc + // algo: + // (!A intersect B) union (A minus B) + + nota := a.invert() + + bs := []interval16{newInterval16Range(uint16(firstOfRange), uint16(endx-1))} + b := newRunContainer16TakeOwnership(bs) + + notAintersectB := nota.intersect(b) + + aMinusB := a.AndNotRunContainer16(b) + + rc2 := notAintersectB.union(aMinusB) + return rc2 +} + +// equals is now logical equals; it does not require the +// same underlying container type. +func (rc *runContainer16) equals(o container) bool { + srb, ok := o.(*runContainer16) + + if !ok { + // maybe value instead of pointer + val, valok := o.(*runContainer16) + if valok { + srb = val + ok = true + } + } + if ok { + // Check if the containers are the same object. + if rc == srb { + return true + } + + if len(srb.iv) != len(rc.iv) { + return false + } + + for i, v := range rc.iv { + if v != srb.iv[i] { + return false + } + } + return true + } + + // use generic comparison + if o.getCardinality() != rc.getCardinality() { + return false + } + rit := rc.getShortIterator() + bit := o.getShortIterator() + + //k := 0 + for rit.hasNext() { + if bit.next() != rit.next() { + return false + } + //k++ + } + return true +} + +func (rc *runContainer16) iaddReturnMinimized(x uint16) container { + rc.Add(x) + return rc +} + +func (rc *runContainer16) iadd(x uint16) (wasNew bool) { + return rc.Add(x) +} + +func (rc *runContainer16) iremoveReturnMinimized(x uint16) container { + rc.removeKey(x) + return rc +} + +func (rc *runContainer16) iremove(x uint16) bool { + return rc.removeKey(x) +} + +func (rc *runContainer16) or(a container) container { + if rc.isFull() { + return rc.clone() + } + switch c := a.(type) { + case *runContainer16: + return rc.union(c) + case *arrayContainer: + return rc.orArray(c) + case *bitmapContainer: + return rc.orBitmapContainer(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) orCardinality(a container) int { + switch c := a.(type) { + case *runContainer16: + return int(rc.unionCardinality(c)) + case *arrayContainer: + return rc.orArrayCardinality(c) + case *bitmapContainer: + return rc.orBitmapContainerCardinality(c) + } + panic("unsupported container type") +} + +// orBitmapContainer finds the union of rc and bc. +func (rc *runContainer16) orBitmapContainer(bc *bitmapContainer) container { + bc2 := newBitmapContainerFromRun(rc) + return bc2.iorBitmap(bc) +} + +func (rc *runContainer16) andBitmapContainerCardinality(bc *bitmapContainer) int { + answer := 0 + for i := range rc.iv { + answer += bc.getCardinalityInRange(uint(rc.iv[i].start), uint(rc.iv[i].last())+1) + } + //bc.computeCardinality() + return answer +} + +func (rc *runContainer16) orBitmapContainerCardinality(bc *bitmapContainer) int { + return rc.getCardinality() + bc.getCardinality() - rc.andBitmapContainerCardinality(bc) +} + +// orArray finds the union of rc and ac. +func (rc *runContainer16) orArray(ac *arrayContainer) container { + bc1 := newBitmapContainerFromRun(rc) + bc2 := ac.toBitmapContainer() + return bc1.orBitmap(bc2) +} + +// orArray finds the union of rc and ac. +func (rc *runContainer16) orArrayCardinality(ac *arrayContainer) int { + return ac.getCardinality() + rc.getCardinality() - rc.andArrayCardinality(ac) +} + +func (rc *runContainer16) ior(a container) container { + if rc.isFull() { + return rc + } + switch c := a.(type) { + case *runContainer16: + return rc.inplaceUnion(c) + case *arrayContainer: + return rc.iorArray(c) + case *bitmapContainer: + return rc.iorBitmapContainer(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) inplaceUnion(rc2 *runContainer16) container { + p("rc.inplaceUnion with len(rc2.iv)=%v", len(rc2.iv)) + for _, p := range rc2.iv { + last := int64(p.last()) + for i := int64(p.start); i <= last; i++ { + rc.Add(uint16(i)) + } + } + return rc +} + +func (rc *runContainer16) iorBitmapContainer(bc *bitmapContainer) container { + + it := bc.getShortIterator() + for it.hasNext() { + rc.Add(it.next()) + } + return rc +} + +func (rc *runContainer16) iorArray(ac *arrayContainer) container { + it := ac.getShortIterator() + for it.hasNext() { + rc.Add(it.next()) + } + return rc +} + +// lazyIOR is described (not yet implemented) in +// this nice note from @lemire on +// https://github.com/RoaringBitmap/roaring/pull/70#issuecomment-263613737 +// +// Description of lazyOR and lazyIOR from @lemire: +// +// Lazy functions are optional and can be simply +// wrapper around non-lazy functions. +// +// The idea of "laziness" is as follows. It is +// inspired by the concept of lazy evaluation +// you might be familiar with (functional programming +// and all that). So a roaring bitmap is +// such that all its containers are, in some +// sense, chosen to use as little memory as +// possible. This is nice. Also, all bitsets +// are "cardinality aware" so that you can do +// fast rank/select queries, or query the +// cardinality of the whole bitmap... very fast, +// without latency. +// +// However, imagine that you are aggregating 100 +// bitmaps together. So you OR the first two, then OR +// that with the third one and so forth. Clearly, +// intermediate bitmaps don't need to be as +// compressed as possible, right? They can be +// in a "dirty state". You only need the end +// result to be in a nice state... which you +// can achieve by calling repairAfterLazy at the end. +// +// The Java/C code does something special for +// the in-place lazy OR runs. The idea is that +// instead of taking two run containers and +// generating a new one, we actually try to +// do the computation in-place through a +// technique invented by @gssiyankai (pinging him!). +// What you do is you check whether the host +// run container has lots of extra capacity. +// If it does, you move its data at the end of +// the backing array, and then you write +// the answer at the beginning. What this +// trick does is minimize memory allocations. +// +func (rc *runContainer16) lazyIOR(a container) container { + // not lazy at the moment + // TODO: make it lazy + return rc.ior(a) + + /* + switch c := a.(type) { + case *arrayContainer: + return rc.lazyIorArray(c) + case *bitmapContainer: + return rc.lazyIorBitmap(c) + case *runContainer16: + return rc.lazyIorRun16(c) + } + panic("unsupported container type") + */ +} + +// lazyOR is described above in lazyIOR. +func (rc *runContainer16) lazyOR(a container) container { + + // not lazy at the moment + // TODO: make it lazy + return rc.or(a) + + /* + switch c := a.(type) { + case *arrayContainer: + return rc.lazyOrArray(c) + case *bitmapContainer: + return rc.lazyOrBitmap(c) + case *runContainer16: + return rc.lazyOrRunContainer16(c) + } + panic("unsupported container type") + */ +} + +func (rc *runContainer16) intersects(a container) bool { + // TODO: optimize by doing inplace/less allocation, possibly? + isect := rc.and(a) + return isect.getCardinality() > 0 +} + +func (rc *runContainer16) xor(a container) container { + switch c := a.(type) { + case *arrayContainer: + return rc.xorArray(c) + case *bitmapContainer: + return rc.xorBitmap(c) + case *runContainer16: + return rc.xorRunContainer16(c) + } + panic("unsupported container type") +} + +func (rc *runContainer16) iandNot(a container) container { + switch c := a.(type) { + case *arrayContainer: + return rc.iandNotArray(c) + case *bitmapContainer: + return rc.iandNotBitmap(c) + case *runContainer16: + return rc.iandNotRunContainer16(c) + } + panic("unsupported container type") +} + +// flip the values in the range [firstOfRange,endx) +func (rc *runContainer16) inot(firstOfRange, endx int) container { + if firstOfRange >= endx { + panic(fmt.Sprintf("invalid %v = endx >= firstOfRange = %v", endx, firstOfRange)) + } + // TODO: minimize copies, do it all inplace; not() makes a copy. + rc = rc.Not(firstOfRange, endx) + return rc +} + +func (rc *runContainer16) getCardinality() int { + return int(rc.cardinality()) +} + +func (rc *runContainer16) rank(x uint16) int { + n := int64(len(rc.iv)) + xx := int64(x) + w, already, _ := rc.search(xx, nil) + if w < 0 { + return 0 + } + if !already && w == n-1 { + return rc.getCardinality() + } + var rnk int64 + if !already { + for i := int64(0); i <= w; i++ { + rnk += rc.iv[i].runlen() + } + return int(rnk) + } + for i := int64(0); i < w; i++ { + rnk += rc.iv[i].runlen() + } + rnk += int64(x-rc.iv[w].start) + 1 + return int(rnk) +} + +func (rc *runContainer16) selectInt(x uint16) int { + return rc.selectInt16(x) +} + +func (rc *runContainer16) andNotRunContainer16(b *runContainer16) container { + return rc.AndNotRunContainer16(b) +} + +func (rc *runContainer16) andNotArray(ac *arrayContainer) container { + rcb := rc.toBitmapContainer() + acb := ac.toBitmapContainer() + return rcb.andNotBitmap(acb) +} + +func (rc *runContainer16) andNotBitmap(bc *bitmapContainer) container { + rcb := rc.toBitmapContainer() + return rcb.andNotBitmap(bc) +} + +func (rc *runContainer16) toBitmapContainer() *bitmapContainer { + p("run16 toBitmap starting; rc has %v ranges", len(rc.iv)) + bc := newBitmapContainer() + for i := range rc.iv { + bc.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + bc.computeCardinality() + return bc +} + +func (rc *runContainer16) iandNotRunContainer16(x2 *runContainer16) container { + rcb := rc.toBitmapContainer() + x2b := x2.toBitmapContainer() + rcb.iandNotBitmapSurely(x2b) + // TODO: check size and optimize the return value + // TODO: is inplace modification really required? If not, elide the copy. + rc2 := newRunContainer16FromBitmapContainer(rcb) + *rc = *rc2 + return rc +} + +func (rc *runContainer16) iandNotArray(ac *arrayContainer) container { + rcb := rc.toBitmapContainer() + acb := ac.toBitmapContainer() + rcb.iandNotBitmapSurely(acb) + // TODO: check size and optimize the return value + // TODO: is inplace modification really required? If not, elide the copy. + rc2 := newRunContainer16FromBitmapContainer(rcb) + *rc = *rc2 + return rc +} + +func (rc *runContainer16) iandNotBitmap(bc *bitmapContainer) container { + rcb := rc.toBitmapContainer() + rcb.iandNotBitmapSurely(bc) + // TODO: check size and optimize the return value + // TODO: is inplace modification really required? If not, elide the copy. + rc2 := newRunContainer16FromBitmapContainer(rcb) + *rc = *rc2 + return rc +} + +func (rc *runContainer16) xorRunContainer16(x2 *runContainer16) container { + rcb := rc.toBitmapContainer() + x2b := x2.toBitmapContainer() + return rcb.xorBitmap(x2b) +} + +func (rc *runContainer16) xorArray(ac *arrayContainer) container { + rcb := rc.toBitmapContainer() + acb := ac.toBitmapContainer() + return rcb.xorBitmap(acb) +} + +func (rc *runContainer16) xorBitmap(bc *bitmapContainer) container { + rcb := rc.toBitmapContainer() + return rcb.xorBitmap(bc) +} + +// convert to bitmap or array *if needed* +func (rc *runContainer16) toEfficientContainer() container { + + // runContainer16SerializedSizeInBytes(numRuns) + sizeAsRunContainer := rc.getSizeInBytes() + sizeAsBitmapContainer := bitmapContainerSizeInBytes() + card := int(rc.cardinality()) + sizeAsArrayContainer := arrayContainerSizeInBytes(card) + if sizeAsRunContainer <= minOfInt(sizeAsBitmapContainer, sizeAsArrayContainer) { + return rc + } + if card <= arrayDefaultMaxSize { + return rc.toArrayContainer() + } + bc := newBitmapContainerFromRun(rc) + return bc +} + +func (rc *runContainer16) toArrayContainer() *arrayContainer { + ac := newArrayContainer() + for i := range rc.iv { + ac.iaddRange(int(rc.iv[i].start), int(rc.iv[i].last())+1) + } + return ac +} + +func newRunContainer16FromContainer(c container) *runContainer16 { + + switch x := c.(type) { + case *runContainer16: + return x.Clone() + case *arrayContainer: + return newRunContainer16FromArray(x) + case *bitmapContainer: + return newRunContainer16FromBitmapContainer(x) + } + panic("unsupported container type") +} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaring.go b/vendor/github.com/RoaringBitmap/roaring/roaring.go new file mode 100644 index 00000000..5045a419 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/roaring.go @@ -0,0 +1,1345 @@ +// Package roaring is an implementation of Roaring Bitmaps in Go. +// They provide fast compressed bitmap data structures (also called bitset). +// They are ideally suited to represent sets of integers over +// relatively small ranges. +// See http://roaringbitmap.org for details. +package roaring + +import ( + "bufio" + "bytes" + "encoding/base64" + "fmt" + "io" + "strconv" +) + +// Bitmap represents a compressed bitmap where you can add integers. +type Bitmap struct { + highlowcontainer roaringArray +} + +// ToBase64 serializes a bitmap as Base64 +func (rb *Bitmap) ToBase64() (string, error) { + buf := new(bytes.Buffer) + _, err := rb.WriteTo(buf) + return base64.StdEncoding.EncodeToString(buf.Bytes()), err + +} + +// FromBase64 deserializes a bitmap from Base64 +func (rb *Bitmap) FromBase64(str string) (int64, error) { + data, err := base64.StdEncoding.DecodeString(str) + if err != nil { + return 0, err + } + buf := bytes.NewBuffer(data) + + return rb.ReadFrom(buf) +} + +// WriteTo writes a serialized version of this bitmap to stream. +// The format is compatible with other RoaringBitmap +// implementations (Java, C) and is documented here: +// https://github.com/RoaringBitmap/RoaringFormatSpec +func (rb *Bitmap) WriteTo(stream io.Writer) (int64, error) { + return rb.highlowcontainer.writeTo(stream) +} + +// ToBytes returns an array of bytes corresponding to what is written +// when calling WriteTo +func (rb *Bitmap) ToBytes() ([]byte, error) { + return rb.highlowcontainer.toBytes() +} + +// WriteToMsgpack writes a msgpack2/snappy-streaming compressed serialized +// version of this bitmap to stream. The format is not +// compatible with the WriteTo() format, and is +// experimental: it may produce smaller on disk +// footprint and/or be faster to read, depending +// on your content. Currently only the Go roaring +// implementation supports this format. +func (rb *Bitmap) WriteToMsgpack(stream io.Writer) (int64, error) { + return 0, rb.highlowcontainer.writeToMsgpack(stream) +} + +// ReadFrom reads a serialized version of this bitmap from stream. +// The format is compatible with other RoaringBitmap +// implementations (Java, C) and is documented here: +// https://github.com/RoaringBitmap/RoaringFormatSpec +func (rb *Bitmap) ReadFrom(stream io.Reader) (int64, error) { + return rb.highlowcontainer.readFrom(stream) +} + +// FromBuffer creates a bitmap from its serialized version stored in buffer +// +// The format specification is available here: +// https://github.com/RoaringBitmap/RoaringFormatSpec +// +// The provided byte array (buf) is expected to be a constant. +// The function makes the best effort attempt not to copy data. +// You should take care not to modify buff as it will +// likely result in unexpected program behavior. +// +// Resulting bitmaps are effectively immutable in the following sense: +// a copy-on-write marker is used so that when you modify the resulting +// bitmap, copies of selected data (containers) are made. +// You should *not* change the copy-on-write status of the resulting +// bitmaps (SetCopyOnWrite). +// +func (rb *Bitmap) FromBuffer(buf []byte) (int64, error) { + return rb.highlowcontainer.fromBuffer(buf) +} + +// RunOptimize attempts to further compress the runs of consecutive values found in the bitmap +func (rb *Bitmap) RunOptimize() { + rb.highlowcontainer.runOptimize() +} + +// HasRunCompression returns true if the bitmap benefits from run compression +func (rb *Bitmap) HasRunCompression() bool { + return rb.highlowcontainer.hasRunCompression() +} + +// ReadFromMsgpack reads a msgpack2/snappy-streaming serialized +// version of this bitmap from stream. The format is +// expected is that written by the WriteToMsgpack() +// call; see additional notes there. +func (rb *Bitmap) ReadFromMsgpack(stream io.Reader) (int64, error) { + return 0, rb.highlowcontainer.readFromMsgpack(stream) +} + +// MarshalBinary implements the encoding.BinaryMarshaler interface for the bitmap +func (rb *Bitmap) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + writer := bufio.NewWriter(&buf) + _, err := rb.WriteTo(writer) + if err != nil { + return nil, err + } + err = writer.Flush() + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface for the bitmap +func (rb *Bitmap) UnmarshalBinary(data []byte) error { + var buf bytes.Buffer + _, err := buf.Write(data) + if err != nil { + return err + } + reader := bufio.NewReader(&buf) + _, err = rb.ReadFrom(reader) + return err +} + +// NewBitmap creates a new empty Bitmap (see also New) +func NewBitmap() *Bitmap { + return &Bitmap{} +} + +// New creates a new empty Bitmap (same as NewBitmap) +func New() *Bitmap { + return &Bitmap{} +} + +// Clear resets the Bitmap to be logically empty, but may retain +// some memory allocations that may speed up future operations +func (rb *Bitmap) Clear() { + rb.highlowcontainer.clear() +} + +// ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order +func (rb *Bitmap) ToArray() []uint32 { + array := make([]uint32, rb.GetCardinality()) + pos := 0 + pos2 := 0 + + for pos < rb.highlowcontainer.size() { + hs := uint32(rb.highlowcontainer.getKeyAtIndex(pos)) << 16 + c := rb.highlowcontainer.getContainerAtIndex(pos) + pos++ + c.fillLeastSignificant16bits(array, pos2, hs) + pos2 += c.getCardinality() + } + return array +} + +// GetSizeInBytes estimates the memory usage of the Bitmap. Note that this +// might differ slightly from the amount of bytes required for persistent storage +func (rb *Bitmap) GetSizeInBytes() uint64 { + size := uint64(8) + for _, c := range rb.highlowcontainer.containers { + size += uint64(2) + uint64(c.getSizeInBytes()) + } + return size +} + +// GetSerializedSizeInBytes computes the serialized size in bytes +// of the Bitmap. It should correspond to the +// number of bytes written when invoking WriteTo. You can expect +// that this function is much cheaper computationally than WriteTo. +func (rb *Bitmap) GetSerializedSizeInBytes() uint64 { + return rb.highlowcontainer.serializedSizeInBytes() +} + +// BoundSerializedSizeInBytes returns an upper bound on the serialized size in bytes +// assuming that one wants to store "cardinality" integers in [0, universe_size) +func BoundSerializedSizeInBytes(cardinality uint64, universeSize uint64) uint64 { + contnbr := (universeSize + uint64(65535)) / uint64(65536) + if contnbr > cardinality { + contnbr = cardinality + // we can't have more containers than we have values + } + headermax := 8*contnbr + 4 + if 4 > (contnbr+7)/8 { + headermax += 4 + } else { + headermax += (contnbr + 7) / 8 + } + valsarray := uint64(arrayContainerSizeInBytes(int(cardinality))) + valsbitmap := contnbr * uint64(bitmapContainerSizeInBytes()) + valsbest := valsarray + if valsbest > valsbitmap { + valsbest = valsbitmap + } + return valsbest + headermax +} + +// IntIterable allows you to iterate over the values in a Bitmap +type IntIterable interface { + HasNext() bool + Next() uint32 +} + +type intIterator struct { + pos int + hs uint32 + iter shortIterable + highlowcontainer *roaringArray +} + +// HasNext returns true if there are more integers to iterate over +func (ii *intIterator) HasNext() bool { + return ii.pos < ii.highlowcontainer.size() +} + +func (ii *intIterator) init() { + if ii.highlowcontainer.size() > ii.pos { + ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getShortIterator() + ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 + } +} + +// Next returns the next integer +func (ii *intIterator) Next() uint32 { + x := uint32(ii.iter.next()) | ii.hs + if !ii.iter.hasNext() { + ii.pos = ii.pos + 1 + ii.init() + } + return x +} + +func newIntIterator(a *Bitmap) *intIterator { + p := new(intIterator) + p.pos = 0 + p.highlowcontainer = &a.highlowcontainer + p.init() + return p +} + +// ManyIntIterable allows you to iterate over the values in a Bitmap +type ManyIntIterable interface { + // pass in a buffer to fill up with values, returns how many values were returned + NextMany([]uint32) int +} + +type manyIntIterator struct { + pos int + hs uint32 + iter manyIterable + highlowcontainer *roaringArray +} + +func (ii *manyIntIterator) init() { + if ii.highlowcontainer.size() > ii.pos { + ii.iter = ii.highlowcontainer.getContainerAtIndex(ii.pos).getManyIterator() + ii.hs = uint32(ii.highlowcontainer.getKeyAtIndex(ii.pos)) << 16 + } else { + ii.iter = nil + } +} + +func (ii *manyIntIterator) NextMany(buf []uint32) int { + n := 0 + for n < len(buf) { + if ii.iter == nil { + break + } + moreN := ii.iter.nextMany(ii.hs, buf[n:]) + n += moreN + if moreN == 0 { + ii.pos = ii.pos + 1 + ii.init() + } + } + + return n +} + +func newManyIntIterator(a *Bitmap) *manyIntIterator { + p := new(manyIntIterator) + p.pos = 0 + p.highlowcontainer = &a.highlowcontainer + p.init() + return p +} + +// String creates a string representation of the Bitmap +func (rb *Bitmap) String() string { + // inspired by https://github.com/fzandona/goroar/ + var buffer bytes.Buffer + start := []byte("{") + buffer.Write(start) + i := rb.Iterator() + counter := 0 + if i.HasNext() { + counter = counter + 1 + buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10)) + } + for i.HasNext() { + buffer.WriteString(",") + counter = counter + 1 + // to avoid exhausting the memory + if counter > 0x40000 { + buffer.WriteString("...") + break + } + buffer.WriteString(strconv.FormatInt(int64(i.Next()), 10)) + } + buffer.WriteString("}") + return buffer.String() +} + +// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order +func (rb *Bitmap) Iterator() IntIterable { + return newIntIterator(rb) +} + +// Iterator creates a new ManyIntIterable to iterate over the integers contained in the bitmap, in sorted order +func (rb *Bitmap) ManyIterator() ManyIntIterable { + return newManyIntIterator(rb) +} + +// Clone creates a copy of the Bitmap +func (rb *Bitmap) Clone() *Bitmap { + ptr := new(Bitmap) + ptr.highlowcontainer = *rb.highlowcontainer.clone() + return ptr +} + +// Minimum get the smallest value stored in this roaring bitmap, assumes that it is not empty +func (rb *Bitmap) Minimum() uint32 { + return uint32(rb.highlowcontainer.containers[0].minimum()) | (uint32(rb.highlowcontainer.keys[0]) << 16) +} + +// Maximum get the largest value stored in this roaring bitmap, assumes that it is not empty +func (rb *Bitmap) Maximum() uint32 { + lastindex := len(rb.highlowcontainer.containers) - 1 + return uint32(rb.highlowcontainer.containers[lastindex].maximum()) | (uint32(rb.highlowcontainer.keys[lastindex]) << 16) +} + +// Contains returns true if the integer is contained in the bitmap +func (rb *Bitmap) Contains(x uint32) bool { + hb := highbits(x) + c := rb.highlowcontainer.getContainer(hb) + return c != nil && c.contains(lowbits(x)) +} + +// ContainsInt returns true if the integer is contained in the bitmap (this is a convenience method, the parameter is casted to uint32 and Contains is called) +func (rb *Bitmap) ContainsInt(x int) bool { + return rb.Contains(uint32(x)) +} + +// Equals returns true if the two bitmaps contain the same integers +func (rb *Bitmap) Equals(o interface{}) bool { + srb, ok := o.(*Bitmap) + if ok { + return srb.highlowcontainer.equals(rb.highlowcontainer) + } + return false +} + +// Add the integer x to the bitmap +func (rb *Bitmap) Add(x uint32) { + hb := highbits(x) + ra := &rb.highlowcontainer + i := ra.getIndex(hb) + if i >= 0 { + var c container + c = ra.getWritableContainerAtIndex(i).iaddReturnMinimized(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, c) + } else { + newac := newArrayContainer() + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.iaddReturnMinimized(lowbits(x))) + } +} + +// add the integer x to the bitmap, return the container and its index +func (rb *Bitmap) addwithptr(x uint32) (int, container) { + hb := highbits(x) + ra := &rb.highlowcontainer + i := ra.getIndex(hb) + var c container + if i >= 0 { + c = ra.getWritableContainerAtIndex(i).iaddReturnMinimized(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, c) + return i, c + } + newac := newArrayContainer() + c = newac.iaddReturnMinimized(lowbits(x)) + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, c) + return -i - 1, c +} + +// CheckedAdd adds the integer x to the bitmap and return true if it was added (false if the integer was already present) +func (rb *Bitmap) CheckedAdd(x uint32) bool { + // TODO: add unit tests for this method + hb := highbits(x) + i := rb.highlowcontainer.getIndex(hb) + if i >= 0 { + C := rb.highlowcontainer.getWritableContainerAtIndex(i) + oldcard := C.getCardinality() + C = C.iaddReturnMinimized(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, C) + return C.getCardinality() > oldcard + } + newac := newArrayContainer() + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, newac.iaddReturnMinimized(lowbits(x))) + return true + +} + +// AddInt adds the integer x to the bitmap (convenience method: the parameter is casted to uint32 and we call Add) +func (rb *Bitmap) AddInt(x int) { + rb.Add(uint32(x)) +} + +// Remove the integer x from the bitmap +func (rb *Bitmap) Remove(x uint32) { + hb := highbits(x) + i := rb.highlowcontainer.getIndex(hb) + if i >= 0 { + c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveReturnMinimized(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, c) + if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 { + rb.highlowcontainer.removeAtIndex(i) + } + } +} + +// CheckedRemove removes the integer x from the bitmap and return true if the integer was effectively remove (and false if the integer was not present) +func (rb *Bitmap) CheckedRemove(x uint32) bool { + // TODO: add unit tests for this method + hb := highbits(x) + i := rb.highlowcontainer.getIndex(hb) + if i >= 0 { + C := rb.highlowcontainer.getWritableContainerAtIndex(i) + oldcard := C.getCardinality() + C = C.iremoveReturnMinimized(lowbits(x)) + rb.highlowcontainer.setContainerAtIndex(i, C) + if rb.highlowcontainer.getContainerAtIndex(i).getCardinality() == 0 { + rb.highlowcontainer.removeAtIndex(i) + return true + } + return C.getCardinality() < oldcard + } + return false + +} + +// IsEmpty returns true if the Bitmap is empty (it is faster than doing (GetCardinality() == 0)) +func (rb *Bitmap) IsEmpty() bool { + return rb.highlowcontainer.size() == 0 +} + +// GetCardinality returns the number of integers contained in the bitmap +func (rb *Bitmap) GetCardinality() uint64 { + size := uint64(0) + for _, c := range rb.highlowcontainer.containers { + size += uint64(c.getCardinality()) + } + return size +} + +// Rank returns the number of integers that are smaller or equal to x (Rank(infinity) would be GetCardinality()) +func (rb *Bitmap) Rank(x uint32) uint64 { + size := uint64(0) + for i := 0; i < rb.highlowcontainer.size(); i++ { + key := rb.highlowcontainer.getKeyAtIndex(i) + if key > highbits(x) { + return size + } + if key < highbits(x) { + size += uint64(rb.highlowcontainer.getContainerAtIndex(i).getCardinality()) + } else { + return size + uint64(rb.highlowcontainer.getContainerAtIndex(i).rank(lowbits(x))) + } + } + return size +} + +// Select returns the xth integer in the bitmap +func (rb *Bitmap) Select(x uint32) (uint32, error) { + if rb.GetCardinality() <= uint64(x) { + return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality()) + } + + remaining := x + for i := 0; i < rb.highlowcontainer.size(); i++ { + c := rb.highlowcontainer.getContainerAtIndex(i) + if remaining >= uint32(c.getCardinality()) { + remaining -= uint32(c.getCardinality()) + } else { + key := rb.highlowcontainer.getKeyAtIndex(i) + return uint32(key)<<16 + uint32(c.selectInt(uint16(remaining))), nil + } + } + return 0, fmt.Errorf("can't find %dth integer in a bitmap with only %d items", x, rb.GetCardinality()) +} + +// And computes the intersection between two bitmaps and stores the result in the current bitmap +func (rb *Bitmap) And(x2 *Bitmap) { + pos1 := 0 + pos2 := 0 + intersectionsize := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + diff := c1.iand(c2) + if diff.getCardinality() > 0 { + rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false) + intersectionsize++ + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + rb.highlowcontainer.resize(intersectionsize) +} + +// OrCardinality returns the cardinality of the union between two bitmaps, bitmaps are not modified +func (rb *Bitmap) OrCardinality(x2 *Bitmap) uint64 { + pos1 := 0 + pos2 := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + answer := uint64(0) +main: + for { + if (pos1 < length1) && (pos2 < length2) { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality()) + pos1++ + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality()) + pos2++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + // TODO: could be faster if we did not have to materialize the container + answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)).getCardinality()) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + for ; pos1 < length1; pos1++ { + answer += uint64(rb.highlowcontainer.getContainerAtIndex(pos1).getCardinality()) + } + for ; pos2 < length2; pos2++ { + answer += uint64(x2.highlowcontainer.getContainerAtIndex(pos2).getCardinality()) + } + return answer +} + +// AndCardinality returns the cardinality of the intersection between two bitmaps, bitmaps are not modified +func (rb *Bitmap) AndCardinality(x2 *Bitmap) uint64 { + pos1 := 0 + pos2 := 0 + answer := uint64(0) + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + c1 := rb.highlowcontainer.getContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + answer += uint64(c1.andCardinality(c2)) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + return answer +} + +// Intersects checks whether two bitmap intersects, bitmaps are not modified +func (rb *Bitmap) Intersects(x2 *Bitmap) bool { + pos1 := 0 + pos2 := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + c1 := rb.highlowcontainer.getContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + if c1.intersects(c2) { + return true + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + return false +} + +// Xor computes the symmetric difference between two bitmaps and stores the result in the current bitmap +func (rb *Bitmap) Xor(x2 *Bitmap) { + pos1 := 0 + pos2 := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + for { + if (pos1 < length1) && (pos2 < length2) { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + if s1 < s2 { + pos1 = rb.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break + } + } else if s1 > s2 { + c := x2.highlowcontainer.getWritableContainerAtIndex(pos2) + rb.highlowcontainer.insertNewKeyValueAt(pos1, x2.highlowcontainer.getKeyAtIndex(pos2), c) + length1++ + pos1++ + pos2++ + } else { + // TODO: couple be computed in-place for reduced memory usage + c := rb.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2)) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(pos1, c) + pos1++ + } else { + rb.highlowcontainer.removeAtIndex(pos1) + length1-- + } + pos2++ + } + } else { + break + } + } + if pos1 == length1 { + rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } +} + +// Or computes the union between two bitmaps and stores the result in the current bitmap +func (rb *Bitmap) Or(x2 *Bitmap) { + pos1 := 0 + pos2 := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for (pos1 < length1) && (pos2 < length2) { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + pos1++ + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + rb.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone()) + pos1++ + length1++ + pos2++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + rb.highlowcontainer.replaceKeyAndContainerAtIndex(pos1, s1, rb.highlowcontainer.getWritableContainerAtIndex(pos1).ior(x2.highlowcontainer.getContainerAtIndex(pos2)), false) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + if pos1 == length1 { + rb.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } +} + +/*func (rb *Bitmap) Or(x2 *Bitmap) { + results := Or(rb, x2) // Todo: could be computed in-place for reduced memory usage + rb.highlowcontainer = results.highlowcontainer +}*/ + +// AndNot computes the difference between two bitmaps and stores the result in the current bitmap +func (rb *Bitmap) AndNot(x2 *Bitmap) { + pos1 := 0 + pos2 := 0 + intersectionsize := 0 + length1 := rb.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + c1 := rb.highlowcontainer.getWritableContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + diff := c1.iandNot(c2) + if diff.getCardinality() > 0 { + rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, diff, false) + intersectionsize++ + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + c1 := rb.highlowcontainer.getContainerAtIndex(pos1) + mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1) + rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite) + intersectionsize++ + pos1++ + if pos1 == length1 { + break main + } + s1 = rb.highlowcontainer.getKeyAtIndex(pos1) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + // TODO:implement as a copy + for pos1 < length1 { + c1 := rb.highlowcontainer.getContainerAtIndex(pos1) + s1 := rb.highlowcontainer.getKeyAtIndex(pos1) + mustCopyOnWrite := rb.highlowcontainer.needsCopyOnWrite(pos1) + rb.highlowcontainer.replaceKeyAndContainerAtIndex(intersectionsize, s1, c1, mustCopyOnWrite) + intersectionsize++ + pos1++ + } + rb.highlowcontainer.resize(intersectionsize) +} + +// Or computes the union between two bitmaps and returns the result +func Or(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for (pos1 < length1) && (pos2 < length2) { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + + for { + if s1 < s2 { + answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) + pos1++ + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 > s2 { + answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) + pos2++ + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { + + answer.highlowcontainer.appendContainer(s1, x1.highlowcontainer.getContainerAtIndex(pos1).or(x2.highlowcontainer.getContainerAtIndex(pos2)), false) + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + if pos1 == length1 { + answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } else if pos2 == length2 { + answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// And computes the intersection between two bitmaps and returns the result +func And(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() +main: + for pos1 < length1 && pos2 < length2 { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 == s2 { + C := x1.highlowcontainer.getContainerAtIndex(pos1) + C = C.and(x2.highlowcontainer.getContainerAtIndex(pos2)) + + if C.getCardinality() > 0 { + answer.highlowcontainer.appendContainer(s1, C, false) + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else if s1 < s2 { + pos1 = x1.highlowcontainer.advanceUntil(s2, pos1) + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else { // s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } + return answer +} + +// Xor computes the symmetric difference between two bitmaps and returns the result +func Xor(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + for { + if (pos1 < length1) && (pos2 < length2) { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + if s1 < s2 { + answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) + pos1++ + } else if s1 > s2 { + answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2) + pos2++ + } else { + c := x1.highlowcontainer.getContainerAtIndex(pos1).xor(x2.highlowcontainer.getContainerAtIndex(pos2)) + if c.getCardinality() > 0 { + answer.highlowcontainer.appendContainer(s1, c, false) + } + pos1++ + pos2++ + } + } else { + break + } + } + if pos1 == length1 { + answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2) + } else if pos2 == length2 { + answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// AndNot computes the difference between two bitmaps and returns the result +func AndNot(x1, x2 *Bitmap) *Bitmap { + answer := NewBitmap() + pos1 := 0 + pos2 := 0 + length1 := x1.highlowcontainer.size() + length2 := x2.highlowcontainer.size() + +main: + for { + if pos1 < length1 && pos2 < length2 { + s1 := x1.highlowcontainer.getKeyAtIndex(pos1) + s2 := x2.highlowcontainer.getKeyAtIndex(pos2) + for { + if s1 < s2 { + answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1) + pos1++ + if pos1 == length1 { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + } else if s1 == s2 { + c1 := x1.highlowcontainer.getContainerAtIndex(pos1) + c2 := x2.highlowcontainer.getContainerAtIndex(pos2) + diff := c1.andNot(c2) + if diff.getCardinality() > 0 { + answer.highlowcontainer.appendContainer(s1, diff, false) + } + pos1++ + pos2++ + if (pos1 == length1) || (pos2 == length2) { + break main + } + s1 = x1.highlowcontainer.getKeyAtIndex(pos1) + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } else { //s1 > s2 + pos2 = x2.highlowcontainer.advanceUntil(s1, pos2) + if pos2 == length2 { + break main + } + s2 = x2.highlowcontainer.getKeyAtIndex(pos2) + } + } + } else { + break + } + } + if pos2 == length2 { + answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1) + } + return answer +} + +// AddMany add all of the values in dat +func (rb *Bitmap) AddMany(dat []uint32) { + if len(dat) == 0 { + return + } + prev := dat[0] + idx, c := rb.addwithptr(prev) + for _, i := range dat[1:] { + if highbits(prev) == highbits(i) { + c = c.iaddReturnMinimized(lowbits(i)) + rb.highlowcontainer.setContainerAtIndex(idx, c) + } else { + idx, c = rb.addwithptr(i) + } + prev = i + } +} + +// BitmapOf generates a new bitmap filled with the specified integers +func BitmapOf(dat ...uint32) *Bitmap { + ans := NewBitmap() + ans.AddMany(dat) + return ans +} + +// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed, +// and any integer present in the range and not in the bitmap is added. +// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range +// while uint64(0x100000000) cannot be represented as a 32-bit value. +func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { + + if rangeEnd > MaxUint32+1 { + panic("rangeEnd > MaxUint32+1") + } + if rangeStart > MaxUint32+1 { + panic("rangeStart > MaxUint32+1") + } + + if rangeStart >= rangeEnd { + return + } + + hbStart := highbits(uint32(rangeStart)) + lbStart := lowbits(uint32(rangeStart)) + hbLast := highbits(uint32(rangeEnd - 1)) + lbLast := lowbits(uint32(rangeEnd - 1)) + + var max uint32 = maxLowBit + for hb := hbStart; hb <= hbLast; hb++ { + var containerStart uint32 + if hb == hbStart { + containerStart = uint32(lbStart) + } + containerLast := max + if hb == hbLast { + containerLast = uint32(lbLast) + } + + i := rb.highlowcontainer.getIndex(hb) + + if i >= 0 { + c := rb.highlowcontainer.getWritableContainerAtIndex(i).inot(int(containerStart), int(containerLast)+1) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(i, c) + } else { + rb.highlowcontainer.removeAtIndex(i) + } + } else { // *think* the range of ones must never be + // empty. + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast))) + } + } +} + +// FlipInt calls Flip after casting the parameters (convenience method) +func (rb *Bitmap) FlipInt(rangeStart, rangeEnd int) { + rb.Flip(uint64(rangeStart), uint64(rangeEnd)) +} + +// AddRange adds the integers in [rangeStart, rangeEnd) to the bitmap. +// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range +// while uint64(0x100000000) cannot be represented as a 32-bit value. +func (rb *Bitmap) AddRange(rangeStart, rangeEnd uint64) { + if rangeStart >= rangeEnd { + return + } + if rangeEnd-1 > MaxUint32 { + panic("rangeEnd-1 > MaxUint32") + } + hbStart := uint32(highbits(uint32(rangeStart))) + lbStart := uint32(lowbits(uint32(rangeStart))) + hbLast := uint32(highbits(uint32(rangeEnd - 1))) + lbLast := uint32(lowbits(uint32(rangeEnd - 1))) + + var max uint32 = maxLowBit + for hb := uint16(hbStart); hb <= uint16(hbLast); hb++ { + containerStart := uint32(0) + if hb == uint16(hbStart) { + containerStart = lbStart + } + containerLast := max + if hb == uint16(hbLast) { + containerLast = lbLast + } + + i := rb.highlowcontainer.getIndex(hb) + + if i >= 0 { + c := rb.highlowcontainer.getWritableContainerAtIndex(i).iaddRange(int(containerStart), int(containerLast)+1) + rb.highlowcontainer.setContainerAtIndex(i, c) + } else { // *think* the range of ones must never be + // empty. + rb.highlowcontainer.insertNewKeyValueAt(-i-1, hb, rangeOfOnes(int(containerStart), int(containerLast))) + } + } +} + +// RemoveRange removes the integers in [rangeStart, rangeEnd) from the bitmap. +// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range +// while uint64(0x100000000) cannot be represented as a 32-bit value. +func (rb *Bitmap) RemoveRange(rangeStart, rangeEnd uint64) { + if rangeStart >= rangeEnd { + return + } + if rangeEnd-1 > MaxUint32 { + // logically, we should assume that the user wants to + // remove all values from rangeStart to infinity + // see https://github.com/RoaringBitmap/roaring/issues/141 + rangeEnd = uint64(0x100000000) + } + hbStart := uint32(highbits(uint32(rangeStart))) + lbStart := uint32(lowbits(uint32(rangeStart))) + hbLast := uint32(highbits(uint32(rangeEnd - 1))) + lbLast := uint32(lowbits(uint32(rangeEnd - 1))) + + var max uint32 = maxLowBit + + if hbStart == hbLast { + i := rb.highlowcontainer.getIndex(uint16(hbStart)) + if i < 0 { + return + } + c := rb.highlowcontainer.getWritableContainerAtIndex(i).iremoveRange(int(lbStart), int(lbLast+1)) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(i, c) + } else { + rb.highlowcontainer.removeAtIndex(i) + } + return + } + ifirst := rb.highlowcontainer.getIndex(uint16(hbStart)) + ilast := rb.highlowcontainer.getIndex(uint16(hbLast)) + + if ifirst >= 0 { + if lbStart != 0 { + c := rb.highlowcontainer.getWritableContainerAtIndex(ifirst).iremoveRange(int(lbStart), int(max+1)) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(ifirst, c) + ifirst++ + } + } + } else { + ifirst = -ifirst - 1 + } + if ilast >= 0 { + if lbLast != max { + c := rb.highlowcontainer.getWritableContainerAtIndex(ilast).iremoveRange(int(0), int(lbLast+1)) + if c.getCardinality() > 0 { + rb.highlowcontainer.setContainerAtIndex(ilast, c) + } else { + ilast++ + } + } else { + ilast++ + } + } else { + ilast = -ilast - 1 + } + rb.highlowcontainer.removeIndexRange(ifirst, ilast) +} + +// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed, +// and any integer present in the range and not in the bitmap is added, a new bitmap is returned leaving +// the current bitmap unchanged. +// The function uses 64-bit parameters even though a Bitmap stores 32-bit values because it is allowed and meaningful to use [0,uint64(0x100000000)) as a range +// while uint64(0x100000000) cannot be represented as a 32-bit value. +func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap { + if rangeStart >= rangeEnd { + return bm.Clone() + } + + if rangeStart > MaxUint32 { + panic("rangeStart > MaxUint32") + } + if rangeEnd-1 > MaxUint32 { + panic("rangeEnd-1 > MaxUint32") + } + + answer := NewBitmap() + hbStart := highbits(uint32(rangeStart)) + lbStart := lowbits(uint32(rangeStart)) + hbLast := highbits(uint32(rangeEnd - 1)) + lbLast := lowbits(uint32(rangeEnd - 1)) + + // copy the containers before the active area + answer.highlowcontainer.appendCopiesUntil(bm.highlowcontainer, hbStart) + + var max uint32 = maxLowBit + for hb := hbStart; hb <= hbLast; hb++ { + var containerStart uint32 + if hb == hbStart { + containerStart = uint32(lbStart) + } + containerLast := max + if hb == hbLast { + containerLast = uint32(lbLast) + } + + i := bm.highlowcontainer.getIndex(hb) + j := answer.highlowcontainer.getIndex(hb) + + if i >= 0 { + c := bm.highlowcontainer.getContainerAtIndex(i).not(int(containerStart), int(containerLast)+1) + if c.getCardinality() > 0 { + answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, c) + } + + } else { // *think* the range of ones must never be + // empty. + answer.highlowcontainer.insertNewKeyValueAt(-j-1, hb, + rangeOfOnes(int(containerStart), int(containerLast))) + } + } + // copy the containers after the active area. + answer.highlowcontainer.appendCopiesAfter(bm.highlowcontainer, hbLast) + + return answer +} + +// SetCopyOnWrite sets this bitmap to use copy-on-write so that copies are fast and memory conscious +// if the parameter is true, otherwise we leave the default where hard copies are made +// (copy-on-write requires extra care in a threaded context). +// Calling SetCopyOnWrite(true) on a bitmap created with FromBuffer is unsafe. +func (rb *Bitmap) SetCopyOnWrite(val bool) { + rb.highlowcontainer.copyOnWrite = val +} + +// GetCopyOnWrite gets this bitmap's copy-on-write property +func (rb *Bitmap) GetCopyOnWrite() (val bool) { + return rb.highlowcontainer.copyOnWrite +} + +// FlipInt calls Flip after casting the parameters (convenience method) +func FlipInt(bm *Bitmap, rangeStart, rangeEnd int) *Bitmap { + return Flip(bm, uint64(rangeStart), uint64(rangeEnd)) +} + +// Statistics provides details on the container types in use. +type Statistics struct { + Cardinality uint64 + Containers uint64 + + ArrayContainers uint64 + ArrayContainerBytes uint64 + ArrayContainerValues uint64 + + BitmapContainers uint64 + BitmapContainerBytes uint64 + BitmapContainerValues uint64 + + RunContainers uint64 + RunContainerBytes uint64 + RunContainerValues uint64 +} + +// Stats returns details on container type usage in a Statistics struct. +func (rb *Bitmap) Stats() Statistics { + stats := Statistics{} + stats.Containers = uint64(len(rb.highlowcontainer.containers)) + for _, c := range rb.highlowcontainer.containers { + stats.Cardinality += uint64(c.getCardinality()) + + switch c.(type) { + case *arrayContainer: + stats.ArrayContainers++ + stats.ArrayContainerBytes += uint64(c.getSizeInBytes()) + stats.ArrayContainerValues += uint64(c.getCardinality()) + case *bitmapContainer: + stats.BitmapContainers++ + stats.BitmapContainerBytes += uint64(c.getSizeInBytes()) + stats.BitmapContainerValues += uint64(c.getCardinality()) + case *runContainer16: + stats.RunContainers++ + stats.RunContainerBytes += uint64(c.getSizeInBytes()) + stats.RunContainerValues += uint64(c.getCardinality()) + } + } + return stats +} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go new file mode 100644 index 00000000..d9659159 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray.go @@ -0,0 +1,893 @@ +package roaring + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + + snappy "github.com/glycerine/go-unsnap-stream" + "github.com/tinylib/msgp/msgp" +) + +//go:generate msgp -unexported + +type container interface { + clone() container + and(container) container + andCardinality(container) int + iand(container) container // i stands for inplace + andNot(container) container + iandNot(container) container // i stands for inplace + getCardinality() int + // rank returns the number of integers that are + // smaller or equal to x. rank(infinity) would be getCardinality(). + rank(uint16) int + + iadd(x uint16) bool // inplace, returns true if x was new. + iaddReturnMinimized(uint16) container // may change return type to minimize storage. + + //addRange(start, final int) container // range is [firstOfRange,lastOfRange) (unused) + iaddRange(start, endx int) container // i stands for inplace, range is [firstOfRange,endx) + + iremove(x uint16) bool // inplace, returns true if x was present. + iremoveReturnMinimized(uint16) container // may change return type to minimize storage. + + not(start, final int) container // range is [firstOfRange,lastOfRange) + inot(firstOfRange, endx int) container // i stands for inplace, range is [firstOfRange,endx) + xor(r container) container + getShortIterator() shortIterable + getManyIterator() manyIterable + contains(i uint16) bool + maximum() uint16 + minimum() uint16 + + // equals is now logical equals; it does not require the + // same underlying container types, but compares across + // any of the implementations. + equals(r container) bool + + fillLeastSignificant16bits(array []uint32, i int, mask uint32) + or(r container) container + orCardinality(r container) int + isFull() bool + ior(r container) container // i stands for inplace + intersects(r container) bool // whether the two containers intersect + lazyOR(r container) container + lazyIOR(r container) container + getSizeInBytes() int + //removeRange(start, final int) container // range is [firstOfRange,lastOfRange) (unused) + iremoveRange(start, final int) container // i stands for inplace, range is [firstOfRange,lastOfRange) + selectInt(x uint16) int // selectInt returns the xth integer in the container + serializedSizeInBytes() int + readFrom(io.Reader) (int, error) + writeTo(io.Writer) (int, error) + + numberOfRuns() int + toEfficientContainer() container + String() string + containerType() contype +} + +type contype uint8 + +const ( + bitmapContype contype = iota + arrayContype + run16Contype + run32Contype +) + +// careful: range is [firstOfRange,lastOfRange] +func rangeOfOnes(start, last int) container { + if start > MaxUint16 { + panic("rangeOfOnes called with start > MaxUint16") + } + if last > MaxUint16 { + panic("rangeOfOnes called with last > MaxUint16") + } + if start < 0 { + panic("rangeOfOnes called with start < 0") + } + if last < 0 { + panic("rangeOfOnes called with last < 0") + } + return newRunContainer16Range(uint16(start), uint16(last)) +} + +type roaringArray struct { + keys []uint16 + containers []container `msg:"-"` // don't try to serialize directly. + needCopyOnWrite []bool + copyOnWrite bool + + // conserz is used at serialization time + // to serialize containers. Otherwise empty. + conserz []containerSerz +} + +// containerSerz facilitates serializing container (tricky to +// serialize because it is an interface) by providing a +// light wrapper with a type identifier. +type containerSerz struct { + t contype `msg:"t"` // type + r msgp.Raw `msg:"r"` // Raw msgpack of the actual container type +} + +func newRoaringArray() *roaringArray { + return &roaringArray{} +} + +// runOptimize compresses the element containers to minimize space consumed. +// Q: how does this interact with copyOnWrite and needCopyOnWrite? +// A: since we aren't changing the logical content, just the representation, +// we don't bother to check the needCopyOnWrite bits. We replace +// (possibly all) elements of ra.containers in-place with space +// optimized versions. +func (ra *roaringArray) runOptimize() { + for i := range ra.containers { + ra.containers[i] = ra.containers[i].toEfficientContainer() + } +} + +func (ra *roaringArray) appendContainer(key uint16, value container, mustCopyOnWrite bool) { + ra.keys = append(ra.keys, key) + ra.containers = append(ra.containers, value) + ra.needCopyOnWrite = append(ra.needCopyOnWrite, mustCopyOnWrite) +} + +func (ra *roaringArray) appendWithoutCopy(sa roaringArray, startingindex int) { + mustCopyOnWrite := sa.needCopyOnWrite[startingindex] + ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], mustCopyOnWrite) +} + +func (ra *roaringArray) appendCopy(sa roaringArray, startingindex int) { + // cow only if the two request it, or if we already have a lightweight copy + copyonwrite := (ra.copyOnWrite && sa.copyOnWrite) || sa.needsCopyOnWrite(startingindex) + if !copyonwrite { + // since there is no copy-on-write, we need to clone the container (this is important) + ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex].clone(), copyonwrite) + } else { + ra.appendContainer(sa.keys[startingindex], sa.containers[startingindex], copyonwrite) + if !sa.needsCopyOnWrite(startingindex) { + sa.setNeedsCopyOnWrite(startingindex) + } + } +} + +func (ra *roaringArray) appendWithoutCopyMany(sa roaringArray, startingindex, end int) { + for i := startingindex; i < end; i++ { + ra.appendWithoutCopy(sa, i) + } +} + +func (ra *roaringArray) appendCopyMany(sa roaringArray, startingindex, end int) { + for i := startingindex; i < end; i++ { + ra.appendCopy(sa, i) + } +} + +func (ra *roaringArray) appendCopiesUntil(sa roaringArray, stoppingKey uint16) { + // cow only if the two request it, or if we already have a lightweight copy + copyonwrite := ra.copyOnWrite && sa.copyOnWrite + + for i := 0; i < sa.size(); i++ { + if sa.keys[i] >= stoppingKey { + break + } + thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i) + if thiscopyonewrite { + ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite) + if !sa.needsCopyOnWrite(i) { + sa.setNeedsCopyOnWrite(i) + } + + } else { + // since there is no copy-on-write, we need to clone the container (this is important) + ra.appendContainer(sa.keys[i], sa.containers[i].clone(), thiscopyonewrite) + + } + } +} + +func (ra *roaringArray) appendCopiesAfter(sa roaringArray, beforeStart uint16) { + // cow only if the two request it, or if we already have a lightweight copy + copyonwrite := ra.copyOnWrite && sa.copyOnWrite + + startLocation := sa.getIndex(beforeStart) + if startLocation >= 0 { + startLocation++ + } else { + startLocation = -startLocation - 1 + } + + for i := startLocation; i < sa.size(); i++ { + thiscopyonewrite := copyonwrite || sa.needsCopyOnWrite(i) + if thiscopyonewrite { + ra.appendContainer(sa.keys[i], sa.containers[i], thiscopyonewrite) + if !sa.needsCopyOnWrite(i) { + sa.setNeedsCopyOnWrite(i) + } + } else { + // since there is no copy-on-write, we need to clone the container (this is important) + ra.appendContainer(sa.keys[i], sa.containers[i].clone(), thiscopyonewrite) + + } + } +} + +func (ra *roaringArray) removeIndexRange(begin, end int) { + if end <= begin { + return + } + + r := end - begin + + copy(ra.keys[begin:], ra.keys[end:]) + copy(ra.containers[begin:], ra.containers[end:]) + copy(ra.needCopyOnWrite[begin:], ra.needCopyOnWrite[end:]) + + ra.resize(len(ra.keys) - r) +} + +func (ra *roaringArray) resize(newsize int) { + for k := newsize; k < len(ra.containers); k++ { + ra.containers[k] = nil + } + + ra.keys = ra.keys[:newsize] + ra.containers = ra.containers[:newsize] + ra.needCopyOnWrite = ra.needCopyOnWrite[:newsize] +} + +func (ra *roaringArray) clear() { + ra.resize(0) + ra.copyOnWrite = false + ra.conserz = nil +} + +func (ra *roaringArray) clone() *roaringArray { + + sa := roaringArray{} + sa.copyOnWrite = ra.copyOnWrite + + // this is where copyOnWrite is used. + if ra.copyOnWrite { + sa.keys = make([]uint16, len(ra.keys)) + copy(sa.keys, ra.keys) + sa.containers = make([]container, len(ra.containers)) + copy(sa.containers, ra.containers) + sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite)) + + ra.markAllAsNeedingCopyOnWrite() + sa.markAllAsNeedingCopyOnWrite() + + // sa.needCopyOnWrite is shared + } else { + // make a full copy + + sa.keys = make([]uint16, len(ra.keys)) + copy(sa.keys, ra.keys) + + sa.containers = make([]container, len(ra.containers)) + for i := range sa.containers { + sa.containers[i] = ra.containers[i].clone() + } + + sa.needCopyOnWrite = make([]bool, len(ra.needCopyOnWrite)) + } + return &sa +} + +// unused function: +//func (ra *roaringArray) containsKey(x uint16) bool { +// return (ra.binarySearch(0, int64(len(ra.keys)), x) >= 0) +//} + +func (ra *roaringArray) getContainer(x uint16) container { + i := ra.binarySearch(0, int64(len(ra.keys)), x) + if i < 0 { + return nil + } + return ra.containers[i] +} + +func (ra *roaringArray) getContainerAtIndex(i int) container { + return ra.containers[i] +} + +func (ra *roaringArray) getFastContainerAtIndex(i int, needsWriteable bool) container { + c := ra.getContainerAtIndex(i) + switch t := c.(type) { + case *arrayContainer: + c = t.toBitmapContainer() + case *runContainer16: + if !t.isFull() { + c = t.toBitmapContainer() + } + case *bitmapContainer: + if needsWriteable && ra.needCopyOnWrite[i] { + c = ra.containers[i].clone() + } + } + return c +} + +func (ra *roaringArray) getWritableContainerAtIndex(i int) container { + if ra.needCopyOnWrite[i] { + ra.containers[i] = ra.containers[i].clone() + ra.needCopyOnWrite[i] = false + } + return ra.containers[i] +} + +func (ra *roaringArray) getIndex(x uint16) int { + // before the binary search, we optimize for frequent cases + size := len(ra.keys) + if (size == 0) || (ra.keys[size-1] == x) { + return size - 1 + } + return ra.binarySearch(0, int64(size), x) +} + +func (ra *roaringArray) getKeyAtIndex(i int) uint16 { + return ra.keys[i] +} + +func (ra *roaringArray) insertNewKeyValueAt(i int, key uint16, value container) { + ra.keys = append(ra.keys, 0) + ra.containers = append(ra.containers, nil) + + copy(ra.keys[i+1:], ra.keys[i:]) + copy(ra.containers[i+1:], ra.containers[i:]) + + ra.keys[i] = key + ra.containers[i] = value + + ra.needCopyOnWrite = append(ra.needCopyOnWrite, false) + copy(ra.needCopyOnWrite[i+1:], ra.needCopyOnWrite[i:]) + ra.needCopyOnWrite[i] = false +} + +func (ra *roaringArray) remove(key uint16) bool { + i := ra.binarySearch(0, int64(len(ra.keys)), key) + if i >= 0 { // if a new key + ra.removeAtIndex(i) + return true + } + return false +} + +func (ra *roaringArray) removeAtIndex(i int) { + copy(ra.keys[i:], ra.keys[i+1:]) + copy(ra.containers[i:], ra.containers[i+1:]) + + copy(ra.needCopyOnWrite[i:], ra.needCopyOnWrite[i+1:]) + + ra.resize(len(ra.keys) - 1) +} + +func (ra *roaringArray) setContainerAtIndex(i int, c container) { + ra.containers[i] = c +} + +func (ra *roaringArray) replaceKeyAndContainerAtIndex(i int, key uint16, c container, mustCopyOnWrite bool) { + ra.keys[i] = key + ra.containers[i] = c + ra.needCopyOnWrite[i] = mustCopyOnWrite +} + +func (ra *roaringArray) size() int { + return len(ra.keys) +} + +func (ra *roaringArray) binarySearch(begin, end int64, ikey uint16) int { + low := begin + high := end - 1 + for low+16 <= high { + middleIndex := low + (high-low)/2 // avoid overflow + middleValue := ra.keys[middleIndex] + + if middleValue < ikey { + low = middleIndex + 1 + } else if middleValue > ikey { + high = middleIndex - 1 + } else { + return int(middleIndex) + } + } + for ; low <= high; low++ { + val := ra.keys[low] + if val >= ikey { + if val == ikey { + return int(low) + } + break + } + } + return -int(low + 1) +} + +func (ra *roaringArray) equals(o interface{}) bool { + srb, ok := o.(roaringArray) + if ok { + + if srb.size() != ra.size() { + return false + } + for i, k := range ra.keys { + if k != srb.keys[i] { + return false + } + } + + for i, c := range ra.containers { + if !c.equals(srb.containers[i]) { + return false + } + } + return true + } + return false +} + +func (ra *roaringArray) headerSize() uint64 { + size := uint64(len(ra.keys)) + if ra.hasRunCompression() { + if size < noOffsetThreshold { // for small bitmaps, we omit the offsets + return 4 + (size+7)/8 + 4*size + } + return 4 + (size+7)/8 + 8*size // - 4 because we pack the size with the cookie + } + return 4 + 4 + 8*size + +} + +// should be dirt cheap +func (ra *roaringArray) serializedSizeInBytes() uint64 { + answer := ra.headerSize() + for _, c := range ra.containers { + answer += uint64(c.serializedSizeInBytes()) + } + return answer +} + +// +// spec: https://github.com/RoaringBitmap/RoaringFormatSpec +// +func (ra *roaringArray) toBytes() ([]byte, error) { + stream := &bytes.Buffer{} + hasRun := ra.hasRunCompression() + isRunSizeInBytes := 0 + cookieSize := 8 + if hasRun { + cookieSize = 4 + isRunSizeInBytes = (len(ra.keys) + 7) / 8 + } + descriptiveHeaderSize := 4 * len(ra.keys) + preambleSize := cookieSize + isRunSizeInBytes + descriptiveHeaderSize + + buf := make([]byte, preambleSize+4*len(ra.keys)) + + nw := 0 + + if hasRun { + binary.LittleEndian.PutUint16(buf[0:], uint16(serialCookie)) + nw += 2 + binary.LittleEndian.PutUint16(buf[2:], uint16(len(ra.keys)-1)) + nw += 2 + + // compute isRun bitmap + var ir []byte + + isRun := newBitmapContainer() + for i, c := range ra.containers { + switch c.(type) { + case *runContainer16: + isRun.iadd(uint16(i)) + } + } + // convert to little endian + ir = isRun.asLittleEndianByteSlice()[:isRunSizeInBytes] + nw += copy(buf[nw:], ir) + } else { + binary.LittleEndian.PutUint32(buf[0:], uint32(serialCookieNoRunContainer)) + nw += 4 + binary.LittleEndian.PutUint32(buf[4:], uint32(len(ra.keys))) + nw += 4 + } + + // descriptive header + for i, key := range ra.keys { + binary.LittleEndian.PutUint16(buf[nw:], key) + nw += 2 + c := ra.containers[i] + binary.LittleEndian.PutUint16(buf[nw:], uint16(c.getCardinality()-1)) + nw += 2 + } + + startOffset := int64(preambleSize + 4*len(ra.keys)) + if !hasRun || (len(ra.keys) >= noOffsetThreshold) { + // offset header + for _, c := range ra.containers { + binary.LittleEndian.PutUint32(buf[nw:], uint32(startOffset)) + nw += 4 + switch rc := c.(type) { + case *runContainer16: + startOffset += 2 + int64(len(rc.iv))*4 + default: + startOffset += int64(getSizeInBytesFromCardinality(c.getCardinality())) + } + } + } + + _, err := stream.Write(buf[:nw]) + if err != nil { + return nil, err + } + for i, c := range ra.containers { + _ = i + _, err := c.writeTo(stream) + if err != nil { + return nil, err + } + } + return stream.Bytes(), nil +} + +// +// spec: https://github.com/RoaringBitmap/RoaringFormatSpec +// +func (ra *roaringArray) writeTo(out io.Writer) (int64, error) { + by, err := ra.toBytes() + if err != nil { + return 0, err + } + n, err := out.Write(by) + if err == nil && n < len(by) { + err = io.ErrShortWrite + } + return int64(n), err +} + +func (ra *roaringArray) fromBuffer(buf []byte) (int64, error) { + pos := 0 + if len(buf) < 8 { + return 0, fmt.Errorf("buffer too small, expecting at least 8 bytes, was %d", len(buf)) + } + + cookie := binary.LittleEndian.Uint32(buf) + pos += 4 + var size uint32 // number of containers + haveRunContainers := false + var isRunBitmap []byte + + // cookie header + if cookie&0x0000FFFF == serialCookie { + haveRunContainers = true + size = uint32(uint16(cookie>>16) + 1) // number of containers + + // create is-run-container bitmap + isRunBitmapSize := (int(size) + 7) / 8 + if pos+isRunBitmapSize > len(buf) { + return 0, fmt.Errorf("malformed bitmap, is-run bitmap overruns buffer at %d", pos+isRunBitmapSize) + } + + isRunBitmap = buf[pos : pos+isRunBitmapSize] + pos += isRunBitmapSize + } else if cookie == serialCookieNoRunContainer { + size = binary.LittleEndian.Uint32(buf[pos:]) + pos += 4 + } else { + return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header") + } + if size > (1 << 16) { + return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.") + } + // descriptive header + // keycard - is {key, cardinality} tuple slice + if pos+2*2*int(size) > len(buf) { + return 0, fmt.Errorf("malfomred bitmap, key-cardinality slice overruns buffer at %d", pos+2*2*int(size)) + } + keycard := byteSliceAsUint16Slice(buf[pos : pos+2*2*int(size)]) + pos += 2 * 2 * int(size) + + if !haveRunContainers || size >= noOffsetThreshold { + pos += 4 * int(size) + } + + // Allocate slices upfront as number of containers is known + if cap(ra.containers) >= int(size) { + ra.containers = ra.containers[:size] + } else { + ra.containers = make([]container, size) + } + if cap(ra.keys) >= int(size) { + ra.keys = ra.keys[:size] + } else { + ra.keys = make([]uint16, size) + } + if cap(ra.needCopyOnWrite) >= int(size) { + ra.needCopyOnWrite = ra.needCopyOnWrite[:size] + } else { + ra.needCopyOnWrite = make([]bool, size) + } + + for i := uint32(0); i < size; i++ { + key := uint16(keycard[2*i]) + card := int(keycard[2*i+1]) + 1 + ra.keys[i] = key + ra.needCopyOnWrite[i] = true + + if haveRunContainers && isRunBitmap[i/8]&(1<<(i%8)) != 0 { + // run container + nr := binary.LittleEndian.Uint16(buf[pos:]) + pos += 2 + if pos+int(nr)*4 > len(buf) { + return 0, fmt.Errorf("malformed bitmap, a run container overruns buffer at %d:%d", pos, pos+int(nr)*4) + } + nb := runContainer16{ + iv: byteSliceAsInterval16Slice(buf[pos : pos+int(nr)*4]), + card: int64(card), + } + pos += int(nr) * 4 + ra.containers[i] = &nb + } else if card > arrayDefaultMaxSize { + // bitmap container + nb := bitmapContainer{ + cardinality: card, + bitmap: byteSliceAsUint64Slice(buf[pos : pos+arrayDefaultMaxSize*2]), + } + pos += arrayDefaultMaxSize * 2 + ra.containers[i] = &nb + } else { + // array container + nb := arrayContainer{ + byteSliceAsUint16Slice(buf[pos : pos+card*2]), + } + pos += card * 2 + ra.containers[i] = &nb + } + } + + return int64(pos), nil +} + +func (ra *roaringArray) readFrom(stream io.Reader) (int64, error) { + pos := 0 + var cookie uint32 + err := binary.Read(stream, binary.LittleEndian, &cookie) + if err != nil { + return 0, fmt.Errorf("error in roaringArray.readFrom: could not read initial cookie: %s", err) + } + pos += 4 + var size uint32 + haveRunContainers := false + var isRun *bitmapContainer + if cookie&0x0000FFFF == serialCookie { + haveRunContainers = true + size = uint32(uint16(cookie>>16) + 1) + bytesToRead := (int(size) + 7) / 8 + numwords := (bytesToRead + 7) / 8 + by := make([]byte, bytesToRead, numwords*8) + nr, err := io.ReadFull(stream, by) + if err != nil { + return 8 + int64(nr), fmt.Errorf("error in readFrom: could not read the "+ + "runContainer bit flags of length %v bytes: %v", bytesToRead, err) + } + pos += bytesToRead + by = by[:cap(by)] + isRun = newBitmapContainer() + for i := 0; i < numwords; i++ { + isRun.bitmap[i] = binary.LittleEndian.Uint64(by) + by = by[8:] + } + } else if cookie == serialCookieNoRunContainer { + err = binary.Read(stream, binary.LittleEndian, &size) + if err != nil { + return 0, fmt.Errorf("error in roaringArray.readFrom: when reading size, got: %s", err) + } + pos += 4 + } else { + return 0, fmt.Errorf("error in roaringArray.readFrom: did not find expected serialCookie in header") + } + if size > (1 << 16) { + return 0, fmt.Errorf("It is logically impossible to have more than (1<<16) containers.") + } + // descriptive header + keycard := make([]uint16, 2*size, 2*size) + err = binary.Read(stream, binary.LittleEndian, keycard) + if err != nil { + return 0, err + } + pos += 2 * 2 * int(size) + // offset header + if !haveRunContainers || size >= noOffsetThreshold { + io.CopyN(ioutil.Discard, stream, 4*int64(size)) // we never skip ahead so this data can be ignored + pos += 4 * int(size) + } + for i := uint32(0); i < size; i++ { + key := int(keycard[2*i]) + card := int(keycard[2*i+1]) + 1 + if haveRunContainers && isRun.contains(uint16(i)) { + nb := newRunContainer16() + nr, err := nb.readFrom(stream) + if err != nil { + return 0, err + } + pos += nr + ra.appendContainer(uint16(key), nb, false) + } else if card > arrayDefaultMaxSize { + nb := newBitmapContainer() + nr, err := nb.readFrom(stream) + if err != nil { + return 0, err + } + nb.cardinality = card + pos += nr + ra.appendContainer(keycard[2*i], nb, false) + } else { + nb := newArrayContainerSize(card) + nr, err := nb.readFrom(stream) + if err != nil { + return 0, err + } + pos += nr + ra.appendContainer(keycard[2*i], nb, false) + } + } + return int64(pos), nil +} + +func (ra *roaringArray) hasRunCompression() bool { + for _, c := range ra.containers { + switch c.(type) { + case *runContainer16: + return true + } + } + return false +} + +func (ra *roaringArray) writeToMsgpack(stream io.Writer) error { + + ra.conserz = make([]containerSerz, len(ra.containers)) + for i, v := range ra.containers { + switch cn := v.(type) { + case *bitmapContainer: + bts, err := cn.MarshalMsg(nil) + if err != nil { + return err + } + ra.conserz[i].t = bitmapContype + ra.conserz[i].r = bts + case *arrayContainer: + bts, err := cn.MarshalMsg(nil) + if err != nil { + return err + } + ra.conserz[i].t = arrayContype + ra.conserz[i].r = bts + case *runContainer16: + bts, err := cn.MarshalMsg(nil) + if err != nil { + return err + } + ra.conserz[i].t = run16Contype + ra.conserz[i].r = bts + default: + panic(fmt.Errorf("Unrecognized container implementation: %T", cn)) + } + } + w := snappy.NewWriter(stream) + err := msgp.Encode(w, ra) + ra.conserz = nil + return err +} + +func (ra *roaringArray) readFromMsgpack(stream io.Reader) error { + r := snappy.NewReader(stream) + err := msgp.Decode(r, ra) + if err != nil { + return err + } + + if len(ra.containers) != len(ra.keys) { + ra.containers = make([]container, len(ra.keys)) + } + + for i, v := range ra.conserz { + switch v.t { + case bitmapContype: + c := &bitmapContainer{} + _, err = c.UnmarshalMsg(v.r) + if err != nil { + return err + } + ra.containers[i] = c + case arrayContype: + c := &arrayContainer{} + _, err = c.UnmarshalMsg(v.r) + if err != nil { + return err + } + ra.containers[i] = c + case run16Contype: + c := &runContainer16{} + _, err = c.UnmarshalMsg(v.r) + if err != nil { + return err + } + ra.containers[i] = c + default: + return fmt.Errorf("unrecognized contype serialization code: '%v'", v.t) + } + } + ra.conserz = nil + return nil +} + +func (ra *roaringArray) advanceUntil(min uint16, pos int) int { + lower := pos + 1 + + if lower >= len(ra.keys) || ra.keys[lower] >= min { + return lower + } + + spansize := 1 + + for lower+spansize < len(ra.keys) && ra.keys[lower+spansize] < min { + spansize *= 2 + } + var upper int + if lower+spansize < len(ra.keys) { + upper = lower + spansize + } else { + upper = len(ra.keys) - 1 + } + + if ra.keys[upper] == min { + return upper + } + + if ra.keys[upper] < min { + // means + // array + // has no + // item + // >= min + // pos = array.length; + return len(ra.keys) + } + + // we know that the next-smallest span was too small + lower += (spansize >> 1) + + mid := 0 + for lower+1 != upper { + mid = (lower + upper) >> 1 + if ra.keys[mid] == min { + return mid + } else if ra.keys[mid] < min { + lower = mid + } else { + upper = mid + } + } + return upper +} + +func (ra *roaringArray) markAllAsNeedingCopyOnWrite() { + for i := range ra.needCopyOnWrite { + ra.needCopyOnWrite[i] = true + } +} + +func (ra *roaringArray) needsCopyOnWrite(i int) bool { + return ra.needCopyOnWrite[i] +} + +func (ra *roaringArray) setNeedsCopyOnWrite(i int) { + ra.needCopyOnWrite[i] = true +} diff --git a/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go new file mode 100644 index 00000000..99fb0f69 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/roaringarray_gen.go @@ -0,0 +1,529 @@ +package roaring + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *containerSerz) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zxvk uint32 + zxvk, err = dc.ReadMapHeader() + if err != nil { + return + } + for zxvk > 0 { + zxvk-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "t": + { + var zbzg uint8 + zbzg, err = dc.ReadUint8() + z.t = contype(zbzg) + } + if err != nil { + return + } + case "r": + err = z.r.DecodeMsg(dc) + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *containerSerz) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "t" + err = en.Append(0x82, 0xa1, 0x74) + if err != nil { + return err + } + err = en.WriteUint8(uint8(z.t)) + if err != nil { + return + } + // write "r" + err = en.Append(0xa1, 0x72) + if err != nil { + return err + } + err = z.r.EncodeMsg(en) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *containerSerz) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "t" + o = append(o, 0x82, 0xa1, 0x74) + o = msgp.AppendUint8(o, uint8(z.t)) + // string "r" + o = append(o, 0xa1, 0x72) + o, err = z.r.MarshalMsg(o) + if err != nil { + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *containerSerz) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zbai uint32 + zbai, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zbai > 0 { + zbai-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "t": + { + var zcmr uint8 + zcmr, bts, err = msgp.ReadUint8Bytes(bts) + z.t = contype(zcmr) + } + if err != nil { + return + } + case "r": + bts, err = z.r.UnmarshalMsg(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *containerSerz) Msgsize() (s int) { + s = 1 + 2 + msgp.Uint8Size + 2 + z.r.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *contype) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zajw uint8 + zajw, err = dc.ReadUint8() + (*z) = contype(zajw) + } + if err != nil { + return + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z contype) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z contype) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *contype) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zwht uint8 + zwht, bts, err = msgp.ReadUint8Bytes(bts) + (*z) = contype(zwht) + } + if err != nil { + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z contype) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *roaringArray) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zlqf uint32 + zlqf, err = dc.ReadMapHeader() + if err != nil { + return + } + for zlqf > 0 { + zlqf-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "keys": + var zdaf uint32 + zdaf, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.keys) >= int(zdaf) { + z.keys = (z.keys)[:zdaf] + } else { + z.keys = make([]uint16, zdaf) + } + for zhct := range z.keys { + z.keys[zhct], err = dc.ReadUint16() + if err != nil { + return + } + } + case "needCopyOnWrite": + var zpks uint32 + zpks, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.needCopyOnWrite) >= int(zpks) { + z.needCopyOnWrite = (z.needCopyOnWrite)[:zpks] + } else { + z.needCopyOnWrite = make([]bool, zpks) + } + for zcua := range z.needCopyOnWrite { + z.needCopyOnWrite[zcua], err = dc.ReadBool() + if err != nil { + return + } + } + case "copyOnWrite": + z.copyOnWrite, err = dc.ReadBool() + if err != nil { + return + } + case "conserz": + var zjfb uint32 + zjfb, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.conserz) >= int(zjfb) { + z.conserz = (z.conserz)[:zjfb] + } else { + z.conserz = make([]containerSerz, zjfb) + } + for zxhx := range z.conserz { + var zcxo uint32 + zcxo, err = dc.ReadMapHeader() + if err != nil { + return + } + for zcxo > 0 { + zcxo-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "t": + { + var zeff uint8 + zeff, err = dc.ReadUint8() + z.conserz[zxhx].t = contype(zeff) + } + if err != nil { + return + } + case "r": + err = z.conserz[zxhx].r.DecodeMsg(dc) + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *roaringArray) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "keys" + err = en.Append(0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.keys))) + if err != nil { + return + } + for zhct := range z.keys { + err = en.WriteUint16(z.keys[zhct]) + if err != nil { + return + } + } + // write "needCopyOnWrite" + err = en.Append(0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.needCopyOnWrite))) + if err != nil { + return + } + for zcua := range z.needCopyOnWrite { + err = en.WriteBool(z.needCopyOnWrite[zcua]) + if err != nil { + return + } + } + // write "copyOnWrite" + err = en.Append(0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65) + if err != nil { + return err + } + err = en.WriteBool(z.copyOnWrite) + if err != nil { + return + } + // write "conserz" + err = en.Append(0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.conserz))) + if err != nil { + return + } + for zxhx := range z.conserz { + // map header, size 2 + // write "t" + err = en.Append(0x82, 0xa1, 0x74) + if err != nil { + return err + } + err = en.WriteUint8(uint8(z.conserz[zxhx].t)) + if err != nil { + return + } + // write "r" + err = en.Append(0xa1, 0x72) + if err != nil { + return err + } + err = z.conserz[zxhx].r.EncodeMsg(en) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *roaringArray) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "keys" + o = append(o, 0x84, 0xa4, 0x6b, 0x65, 0x79, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.keys))) + for zhct := range z.keys { + o = msgp.AppendUint16(o, z.keys[zhct]) + } + // string "needCopyOnWrite" + o = append(o, 0xaf, 0x6e, 0x65, 0x65, 0x64, 0x43, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65) + o = msgp.AppendArrayHeader(o, uint32(len(z.needCopyOnWrite))) + for zcua := range z.needCopyOnWrite { + o = msgp.AppendBool(o, z.needCopyOnWrite[zcua]) + } + // string "copyOnWrite" + o = append(o, 0xab, 0x63, 0x6f, 0x70, 0x79, 0x4f, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65) + o = msgp.AppendBool(o, z.copyOnWrite) + // string "conserz" + o = append(o, 0xa7, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x72, 0x7a) + o = msgp.AppendArrayHeader(o, uint32(len(z.conserz))) + for zxhx := range z.conserz { + // map header, size 2 + // string "t" + o = append(o, 0x82, 0xa1, 0x74) + o = msgp.AppendUint8(o, uint8(z.conserz[zxhx].t)) + // string "r" + o = append(o, 0xa1, 0x72) + o, err = z.conserz[zxhx].r.MarshalMsg(o) + if err != nil { + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *roaringArray) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zrsw uint32 + zrsw, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zrsw > 0 { + zrsw-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "keys": + var zxpk uint32 + zxpk, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.keys) >= int(zxpk) { + z.keys = (z.keys)[:zxpk] + } else { + z.keys = make([]uint16, zxpk) + } + for zhct := range z.keys { + z.keys[zhct], bts, err = msgp.ReadUint16Bytes(bts) + if err != nil { + return + } + } + case "needCopyOnWrite": + var zdnj uint32 + zdnj, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.needCopyOnWrite) >= int(zdnj) { + z.needCopyOnWrite = (z.needCopyOnWrite)[:zdnj] + } else { + z.needCopyOnWrite = make([]bool, zdnj) + } + for zcua := range z.needCopyOnWrite { + z.needCopyOnWrite[zcua], bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + return + } + } + case "copyOnWrite": + z.copyOnWrite, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + return + } + case "conserz": + var zobc uint32 + zobc, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.conserz) >= int(zobc) { + z.conserz = (z.conserz)[:zobc] + } else { + z.conserz = make([]containerSerz, zobc) + } + for zxhx := range z.conserz { + var zsnv uint32 + zsnv, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zsnv > 0 { + zsnv-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "t": + { + var zkgt uint8 + zkgt, bts, err = msgp.ReadUint8Bytes(bts) + z.conserz[zxhx].t = contype(zkgt) + } + if err != nil { + return + } + case "r": + bts, err = z.conserz[zxhx].r.UnmarshalMsg(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *roaringArray) Msgsize() (s int) { + s = 1 + 5 + msgp.ArrayHeaderSize + (len(z.keys) * (msgp.Uint16Size)) + 16 + msgp.ArrayHeaderSize + (len(z.needCopyOnWrite) * (msgp.BoolSize)) + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize + for zxhx := range z.conserz { + s += 1 + 2 + msgp.Uint8Size + 2 + z.conserz[zxhx].r.Msgsize() + } + return +} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization.go b/vendor/github.com/RoaringBitmap/roaring/serialization.go new file mode 100644 index 00000000..59c39a66 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/serialization.go @@ -0,0 +1,83 @@ +package roaring + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/tinylib/msgp/msgp" +) + +// writeTo for runContainer16 follows this +// spec: https://github.com/RoaringBitmap/RoaringFormatSpec +// +func (b *runContainer16) writeTo(stream io.Writer) (int, error) { + buf := make([]byte, 2+4*len(b.iv)) + binary.LittleEndian.PutUint16(buf[0:], uint16(len(b.iv))) + for i, v := range b.iv { + binary.LittleEndian.PutUint16(buf[2+i*4:], v.start) + binary.LittleEndian.PutUint16(buf[2+2+i*4:], v.length) + } + return stream.Write(buf) +} + +func (b *runContainer32) writeToMsgpack(stream io.Writer) (int, error) { + bts, err := b.MarshalMsg(nil) + if err != nil { + return 0, err + } + return stream.Write(bts) +} + +func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) { + bts, err := b.MarshalMsg(nil) + if err != nil { + return 0, err + } + return stream.Write(bts) +} + +func (b *runContainer32) readFromMsgpack(stream io.Reader) (int, error) { + err := msgp.Decode(stream, b) + return 0, err +} + +func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) { + err := msgp.Decode(stream, b) + return 0, err +} + +var errCorruptedStream = errors.New("insufficient/odd number of stored bytes, corrupted stream detected") + +func (b *runContainer16) readFrom(stream io.Reader) (int, error) { + b.iv = b.iv[:0] + b.card = 0 + var numRuns uint16 + err := binary.Read(stream, binary.LittleEndian, &numRuns) + if err != nil { + return 0, err + } + nr := int(numRuns) + encRun := make([]uint16, 2*nr) + by := make([]byte, 4*nr) + err = binary.Read(stream, binary.LittleEndian, &by) + if err != nil { + return 0, err + } + for i := range encRun { + if len(by) < 2 { + return 0, errCorruptedStream + } + encRun[i] = binary.LittleEndian.Uint16(by) + by = by[2:] + } + for i := 0; i < nr; i++ { + if i > 0 && b.iv[i-1].last() >= encRun[i*2] { + return 0, fmt.Errorf("error: stored runContainer had runs that were not in sorted order!! (b.iv[i-1=%v].last = %v >= encRun[i=%v] = %v)", i-1, b.iv[i-1].last(), i, encRun[i*2]) + } + b.iv = append(b.iv, interval16{start: encRun[i*2], length: encRun[i*2+1]}) + b.card += int64(encRun[i*2+1]) + 1 + } + return 0, err +} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go new file mode 100644 index 00000000..7fcef769 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_generic.go @@ -0,0 +1,118 @@ +// +build !amd64,!386 appengine + +package roaring + +import ( + "encoding/binary" + "io" +) + +func (b *arrayContainer) writeTo(stream io.Writer) (int, error) { + buf := make([]byte, 2*len(b.content)) + for i, v := range b.content { + base := i * 2 + buf[base] = byte(v) + buf[base+1] = byte(v >> 8) + } + return stream.Write(buf) +} + +func (b *arrayContainer) readFrom(stream io.Reader) (int, error) { + err := binary.Read(stream, binary.LittleEndian, b.content) + if err != nil { + return 0, err + } + return 2 * len(b.content), nil +} + +func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) { + // Write set + buf := make([]byte, 8*len(b.bitmap)) + for i, v := range b.bitmap { + base := i * 8 + buf[base] = byte(v) + buf[base+1] = byte(v >> 8) + buf[base+2] = byte(v >> 16) + buf[base+3] = byte(v >> 24) + buf[base+4] = byte(v >> 32) + buf[base+5] = byte(v >> 40) + buf[base+6] = byte(v >> 48) + buf[base+7] = byte(v >> 56) + } + return stream.Write(buf) +} + +func (b *bitmapContainer) readFrom(stream io.Reader) (int, error) { + err := binary.Read(stream, binary.LittleEndian, b.bitmap) + if err != nil { + return 0, err + } + b.computeCardinality() + return 8 * len(b.bitmap), nil +} + +func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { + by := make([]byte, len(bc.bitmap)*8) + for i := range bc.bitmap { + binary.LittleEndian.PutUint64(by[i*8:], bc.bitmap[i]) + } + return by +} + +func uint64SliceAsByteSlice(slice []uint64) []byte { + by := make([]byte, len(slice)*8) + + for i, v := range slice { + binary.LittleEndian.PutUint64(by[i*8:], v) + } + + return by +} + +func byteSliceAsUint16Slice(slice []byte) []uint16 { + if len(slice)%2 != 0 { + panic("Slice size should be divisible by 2") + } + + b := make([]uint16, len(slice)/2) + + for i := range b { + b[i] = binary.LittleEndian.Uint16(slice[2*i:]) + } + + return b +} + +func byteSliceAsUint64Slice(slice []byte) []uint64 { + if len(slice)%8 != 0 { + panic("Slice size should be divisible by 8") + } + + b := make([]uint64, len(slice)/8) + + for i := range b { + b[i] = binary.LittleEndian.Uint64(slice[8*i:]) + } + + return b +} + +// Converts a byte slice to a interval16 slice. +// The function assumes that the slice byte buffer is run container data +// encoded according to Roaring Format Spec +func byteSliceAsInterval16Slice(byteSlice []byte) []interval16 { + if len(byteSlice)%4 != 0 { + panic("Slice size should be divisible by 4") + } + + intervalSlice := make([]interval16, len(byteSlice)/4) + + for i := range intervalSlice { + intervalSlice[i] = interval16{ + start: binary.LittleEndian.Uint16(byteSlice[i*4:]), + length: binary.LittleEndian.Uint16(byteSlice[i*4+2:]), + } + } + + return intervalSlice +} diff --git a/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go new file mode 100644 index 00000000..c1d3ad30 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go @@ -0,0 +1,113 @@ +// +build 386 amd64,!appengine + +package roaring + +import ( + "io" + "reflect" + "unsafe" +) + +func (ac *arrayContainer) writeTo(stream io.Writer) (int, error) { + buf := uint16SliceAsByteSlice(ac.content) + return stream.Write(buf) +} + +func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) { + buf := uint64SliceAsByteSlice(bc.bitmap) + return stream.Write(buf) +} + +// readFrom reads an arrayContainer from stream. +// PRE-REQUISITE: you must size the arrayContainer correctly (allocate b.content) +// *before* you call readFrom. We can't guess the size in the stream +// by this point. +func (ac *arrayContainer) readFrom(stream io.Reader) (int, error) { + buf := uint16SliceAsByteSlice(ac.content) + return io.ReadFull(stream, buf) +} + +func (bc *bitmapContainer) readFrom(stream io.Reader) (int, error) { + buf := uint64SliceAsByteSlice(bc.bitmap) + n, err := io.ReadFull(stream, buf) + bc.computeCardinality() + return n, err +} + +func uint64SliceAsByteSlice(slice []uint64) []byte { + // make a new slice header + header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + + // update its capacity and length + header.Len *= 8 + header.Cap *= 8 + + // return it + return *(*[]byte)(unsafe.Pointer(&header)) +} + +func uint16SliceAsByteSlice(slice []uint16) []byte { + // make a new slice header + header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + + // update its capacity and length + header.Len *= 2 + header.Cap *= 2 + + // return it + return *(*[]byte)(unsafe.Pointer(&header)) +} + +func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { + return uint64SliceAsByteSlice(bc.bitmap) +} + +// Deserialization code follows + +func byteSliceAsUint16Slice(slice []byte) []uint16 { + if len(slice)%2 != 0 { + panic("Slice size should be divisible by 2") + } + + // make a new slice header + header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + + // update its capacity and length + header.Len /= 2 + header.Cap /= 2 + + // return it + return *(*[]uint16)(unsafe.Pointer(&header)) +} + +func byteSliceAsUint64Slice(slice []byte) []uint64 { + if len(slice)%8 != 0 { + panic("Slice size should be divisible by 8") + } + + // make a new slice header + header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + + // update its capacity and length + header.Len /= 8 + header.Cap /= 8 + + // return it + return *(*[]uint64)(unsafe.Pointer(&header)) +} + +func byteSliceAsInterval16Slice(slice []byte) []interval16 { + if len(slice)%4 != 0 { + panic("Slice size should be divisible by 4") + } + + // make a new slice header + header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) + + // update its capacity and length + header.Len /= 4 + header.Cap /= 4 + + // return it + return *(*[]interval16)(unsafe.Pointer(&header)) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go b/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go new file mode 100644 index 00000000..5eaa2220 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go @@ -0,0 +1,21 @@ +// +build gofuzz + +package roaring + +import "bytes" + +func FuzzSerializationStream(data []byte) int { + newrb := NewBitmap() + if _, err := newrb.ReadFrom(bytes.NewReader(data)); err != nil { + return 0 + } + return 1 +} + +func FuzzSerializationBuffer(data []byte) int { + newrb := NewBitmap() + if _, err := newrb.FromBuffer(data); err != nil { + return 0 + } + return 1 +} diff --git a/vendor/github.com/RoaringBitmap/roaring/setutil.go b/vendor/github.com/RoaringBitmap/roaring/setutil.go new file mode 100644 index 00000000..3e8c01dd --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/setutil.go @@ -0,0 +1,609 @@ +package roaring + +func equal(a, b []uint16) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func difference(set1 []uint16, set2 []uint16, buffer []uint16) int { + if 0 == len(set2) { + for k := 0; k < len(set1); k++ { + buffer[k] = set1[k] + } + return len(set1) + } + if 0 == len(set1) { + return 0 + } + pos := 0 + k1 := 0 + k2 := 0 + buffer = buffer[:cap(buffer)] + s1 := set1[k1] + s2 := set2[k2] + for { + if s1 < s2 { + buffer[pos] = s1 + pos++ + k1++ + if k1 >= len(set1) { + break + } + s1 = set1[k1] + } else if s1 == s2 { + k1++ + k2++ + if k1 >= len(set1) { + break + } + s1 = set1[k1] + if k2 >= len(set2) { + for ; k1 < len(set1); k1++ { + buffer[pos] = set1[k1] + pos++ + } + break + } + s2 = set2[k2] + } else { // if (val1>val2) + k2++ + if k2 >= len(set2) { + for ; k1 < len(set1); k1++ { + buffer[pos] = set1[k1] + pos++ + } + break + } + s2 = set2[k2] + } + } + return pos + +} + +func exclusiveUnion2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { + if 0 == len(set2) { + buffer = buffer[:len(set1)] + copy(buffer, set1[:]) + return len(set1) + } + if 0 == len(set1) { + buffer = buffer[:len(set2)] + copy(buffer, set2[:]) + return len(set2) + } + pos := 0 + k1 := 0 + k2 := 0 + s1 := set1[k1] + s2 := set2[k2] + buffer = buffer[:cap(buffer)] + for { + if s1 < s2 { + buffer[pos] = s1 + pos++ + k1++ + if k1 >= len(set1) { + for ; k2 < len(set2); k2++ { + buffer[pos] = set2[k2] + pos++ + } + break + } + s1 = set1[k1] + } else if s1 == s2 { + k1++ + k2++ + if k1 >= len(set1) { + for ; k2 < len(set2); k2++ { + buffer[pos] = set2[k2] + pos++ + } + break + } + if k2 >= len(set2) { + for ; k1 < len(set1); k1++ { + buffer[pos] = set1[k1] + pos++ + } + break + } + s1 = set1[k1] + s2 = set2[k2] + } else { // if (val1>val2) + buffer[pos] = s2 + pos++ + k2++ + if k2 >= len(set2) { + for ; k1 < len(set1); k1++ { + buffer[pos] = set1[k1] + pos++ + } + break + } + s2 = set2[k2] + } + } + return pos +} + +func union2by2(set1 []uint16, set2 []uint16, buffer []uint16) int { + pos := 0 + k1 := 0 + k2 := 0 + if 0 == len(set2) { + buffer = buffer[:len(set1)] + copy(buffer, set1[:]) + return len(set1) + } + if 0 == len(set1) { + buffer = buffer[:len(set2)] + copy(buffer, set2[:]) + return len(set2) + } + s1 := set1[k1] + s2 := set2[k2] + buffer = buffer[:cap(buffer)] + for { + if s1 < s2 { + buffer[pos] = s1 + pos++ + k1++ + if k1 >= len(set1) { + copy(buffer[pos:], set2[k2:]) + pos += len(set2) - k2 + break + } + s1 = set1[k1] + } else if s1 == s2 { + buffer[pos] = s1 + pos++ + k1++ + k2++ + if k1 >= len(set1) { + copy(buffer[pos:], set2[k2:]) + pos += len(set2) - k2 + break + } + if k2 >= len(set2) { + copy(buffer[pos:], set1[k1:]) + pos += len(set1) - k1 + break + } + s1 = set1[k1] + s2 = set2[k2] + } else { // if (set1[k1]>set2[k2]) + buffer[pos] = s2 + pos++ + k2++ + if k2 >= len(set2) { + copy(buffer[pos:], set1[k1:]) + pos += len(set1) - k1 + break + } + s2 = set2[k2] + } + } + return pos +} + +func union2by2Cardinality(set1 []uint16, set2 []uint16) int { + pos := 0 + k1 := 0 + k2 := 0 + if 0 == len(set2) { + return len(set1) + } + if 0 == len(set1) { + return len(set2) + } + s1 := set1[k1] + s2 := set2[k2] + for { + if s1 < s2 { + pos++ + k1++ + if k1 >= len(set1) { + pos += len(set2) - k2 + break + } + s1 = set1[k1] + } else if s1 == s2 { + pos++ + k1++ + k2++ + if k1 >= len(set1) { + pos += len(set2) - k2 + break + } + if k2 >= len(set2) { + pos += len(set1) - k1 + break + } + s1 = set1[k1] + s2 = set2[k2] + } else { // if (set1[k1]>set2[k2]) + pos++ + k2++ + if k2 >= len(set2) { + pos += len(set1) - k1 + break + } + s2 = set2[k2] + } + } + return pos +} + +func intersection2by2( + set1 []uint16, + set2 []uint16, + buffer []uint16) int { + + if len(set1)*64 < len(set2) { + return onesidedgallopingintersect2by2(set1, set2, buffer) + } else if len(set2)*64 < len(set1) { + return onesidedgallopingintersect2by2(set2, set1, buffer) + } else { + return localintersect2by2(set1, set2, buffer) + } +} + +func intersection2by2Cardinality( + set1 []uint16, + set2 []uint16) int { + + if len(set1)*64 < len(set2) { + return onesidedgallopingintersect2by2Cardinality(set1, set2) + } else if len(set2)*64 < len(set1) { + return onesidedgallopingintersect2by2Cardinality(set2, set1) + } else { + return localintersect2by2Cardinality(set1, set2) + } +} + +func intersects2by2( + set1 []uint16, + set2 []uint16) bool { + // could be optimized if one set is much larger than the other one + if (0 == len(set1)) || (0 == len(set2)) { + return false + } + k1 := 0 + k2 := 0 + s1 := set1[k1] + s2 := set2[k2] +mainwhile: + for { + + if s2 < s1 { + for { + k2++ + if k2 == len(set2) { + break mainwhile + } + s2 = set2[k2] + if s2 >= s1 { + break + } + } + } + if s1 < s2 { + for { + k1++ + if k1 == len(set1) { + break mainwhile + } + s1 = set1[k1] + if s1 >= s2 { + break + } + } + + } else { + // (set2[k2] == set1[k1]) + return true + } + } + return false +} + +func localintersect2by2( + set1 []uint16, + set2 []uint16, + buffer []uint16) int { + + if (0 == len(set1)) || (0 == len(set2)) { + return 0 + } + k1 := 0 + k2 := 0 + pos := 0 + buffer = buffer[:cap(buffer)] + s1 := set1[k1] + s2 := set2[k2] +mainwhile: + for { + if s2 < s1 { + for { + k2++ + if k2 == len(set2) { + break mainwhile + } + s2 = set2[k2] + if s2 >= s1 { + break + } + } + } + if s1 < s2 { + for { + k1++ + if k1 == len(set1) { + break mainwhile + } + s1 = set1[k1] + if s1 >= s2 { + break + } + } + + } else { + // (set2[k2] == set1[k1]) + buffer[pos] = s1 + pos++ + k1++ + if k1 == len(set1) { + break + } + s1 = set1[k1] + k2++ + if k2 == len(set2) { + break + } + s2 = set2[k2] + } + } + return pos +} + +func localintersect2by2Cardinality( + set1 []uint16, + set2 []uint16) int { + + if (0 == len(set1)) || (0 == len(set2)) { + return 0 + } + k1 := 0 + k2 := 0 + pos := 0 + s1 := set1[k1] + s2 := set2[k2] +mainwhile: + for { + if s2 < s1 { + for { + k2++ + if k2 == len(set2) { + break mainwhile + } + s2 = set2[k2] + if s2 >= s1 { + break + } + } + } + if s1 < s2 { + for { + k1++ + if k1 == len(set1) { + break mainwhile + } + s1 = set1[k1] + if s1 >= s2 { + break + } + } + + } else { + // (set2[k2] == set1[k1]) + pos++ + k1++ + if k1 == len(set1) { + break + } + s1 = set1[k1] + k2++ + if k2 == len(set2) { + break + } + s2 = set2[k2] + } + } + return pos +} + +func advanceUntil( + array []uint16, + pos int, + length int, + min uint16) int { + lower := pos + 1 + + if lower >= length || array[lower] >= min { + return lower + } + + spansize := 1 + + for lower+spansize < length && array[lower+spansize] < min { + spansize *= 2 + } + var upper int + if lower+spansize < length { + upper = lower + spansize + } else { + upper = length - 1 + } + + if array[upper] == min { + return upper + } + + if array[upper] < min { + // means + // array + // has no + // item + // >= min + // pos = array.length; + return length + } + + // we know that the next-smallest span was too small + lower += (spansize >> 1) + + mid := 0 + for lower+1 != upper { + mid = (lower + upper) >> 1 + if array[mid] == min { + return mid + } else if array[mid] < min { + lower = mid + } else { + upper = mid + } + } + return upper + +} + +func onesidedgallopingintersect2by2( + smallset []uint16, + largeset []uint16, + buffer []uint16) int { + + if 0 == len(smallset) { + return 0 + } + buffer = buffer[:cap(buffer)] + k1 := 0 + k2 := 0 + pos := 0 + s1 := largeset[k1] + s2 := smallset[k2] +mainwhile: + + for { + if s1 < s2 { + k1 = advanceUntil(largeset, k1, len(largeset), s2) + if k1 == len(largeset) { + break mainwhile + } + s1 = largeset[k1] + } + if s2 < s1 { + k2++ + if k2 == len(smallset) { + break mainwhile + } + s2 = smallset[k2] + } else { + + buffer[pos] = s2 + pos++ + k2++ + if k2 == len(smallset) { + break + } + s2 = smallset[k2] + k1 = advanceUntil(largeset, k1, len(largeset), s2) + if k1 == len(largeset) { + break mainwhile + } + s1 = largeset[k1] + } + + } + return pos +} + +func onesidedgallopingintersect2by2Cardinality( + smallset []uint16, + largeset []uint16) int { + + if 0 == len(smallset) { + return 0 + } + k1 := 0 + k2 := 0 + pos := 0 + s1 := largeset[k1] + s2 := smallset[k2] +mainwhile: + + for { + if s1 < s2 { + k1 = advanceUntil(largeset, k1, len(largeset), s2) + if k1 == len(largeset) { + break mainwhile + } + s1 = largeset[k1] + } + if s2 < s1 { + k2++ + if k2 == len(smallset) { + break mainwhile + } + s2 = smallset[k2] + } else { + + pos++ + k2++ + if k2 == len(smallset) { + break + } + s2 = smallset[k2] + k1 = advanceUntil(largeset, k1, len(largeset), s2) + if k1 == len(largeset) { + break mainwhile + } + s1 = largeset[k1] + } + + } + return pos +} + +func binarySearch(array []uint16, ikey uint16) int { + low := 0 + high := len(array) - 1 + for low+16 <= high { + middleIndex := int(uint32(low+high) >> 1) + middleValue := array[middleIndex] + if middleValue < ikey { + low = middleIndex + 1 + } else if middleValue > ikey { + high = middleIndex - 1 + } else { + return middleIndex + } + } + for ; low <= high; low++ { + val := array[low] + if val >= ikey { + if val == ikey { + return low + } + break + } + } + return -(low + 1) +} diff --git a/vendor/github.com/RoaringBitmap/roaring/shortiterator.go b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go new file mode 100644 index 00000000..ef0acbd1 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/shortiterator.go @@ -0,0 +1,21 @@ +package roaring + +type shortIterable interface { + hasNext() bool + next() uint16 +} + +type shortIterator struct { + slice []uint16 + loc int +} + +func (si *shortIterator) hasNext() bool { + return si.loc < len(si.slice) +} + +func (si *shortIterator) next() uint16 { + a := si.slice[si.loc] + si.loc++ + return a +} diff --git a/vendor/github.com/RoaringBitmap/roaring/smat.go b/vendor/github.com/RoaringBitmap/roaring/smat.go new file mode 100644 index 00000000..9da47563 --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/smat.go @@ -0,0 +1,383 @@ +// +build gofuzz + +/* +# Instructions for smat testing for roaring + +[smat](https://github.com/mschoch/smat) is a framework that provides +state machine assisted fuzz testing. + +To run the smat tests for roaring... + +## Prerequisites + + $ go get github.com/dvyukov/go-fuzz/go-fuzz + $ go get github.com/dvyukov/go-fuzz/go-fuzz-build + +## Steps + +1. Generate initial smat corpus: +``` + go test -tags=gofuzz -run=TestGenerateSmatCorpus +``` + +2. Build go-fuzz test program with instrumentation: +``` + go-fuzz-build -func FuzzSmat github.com/RoaringBitmap/roaring +``` + +3. Run go-fuzz: +``` + go-fuzz -bin=./roaring-fuzz.zip -workdir=workdir/ -timeout=200 +``` + +You should see output like... +``` +2016/09/16 13:58:35 slaves: 8, corpus: 1 (3s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 3s +2016/09/16 13:58:38 slaves: 8, corpus: 1 (6s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 6s +2016/09/16 13:58:41 slaves: 8, corpus: 1 (9s ago), crashers: 0, restarts: 1/44, execs: 44 (5/sec), cover: 0, uptime: 9s +2016/09/16 13:58:44 slaves: 8, corpus: 1 (12s ago), crashers: 0, restarts: 1/45, execs: 45 (4/sec), cover: 0, uptime: 12s +2016/09/16 13:58:47 slaves: 8, corpus: 1 (15s ago), crashers: 0, restarts: 1/46, execs: 46 (3/sec), cover: 0, uptime: 15s +2016/09/16 13:58:50 slaves: 8, corpus: 1 (18s ago), crashers: 0, restarts: 1/47, execs: 47 (3/sec), cover: 0, uptime: 18s +2016/09/16 13:58:53 slaves: 8, corpus: 1 (21s ago), crashers: 0, restarts: 1/63, execs: 63 (3/sec), cover: 0, uptime: 21s +2016/09/16 13:58:56 slaves: 8, corpus: 1 (24s ago), crashers: 0, restarts: 1/65, execs: 65 (3/sec), cover: 0, uptime: 24s +2016/09/16 13:58:59 slaves: 8, corpus: 1 (27s ago), crashers: 0, restarts: 1/66, execs: 66 (2/sec), cover: 0, uptime: 27s +2016/09/16 13:59:02 slaves: 8, corpus: 1 (30s ago), crashers: 0, restarts: 1/67, execs: 67 (2/sec), cover: 0, uptime: 30s +2016/09/16 13:59:05 slaves: 8, corpus: 1 (33s ago), crashers: 0, restarts: 1/83, execs: 83 (3/sec), cover: 0, uptime: 33s +2016/09/16 13:59:08 slaves: 8, corpus: 1 (36s ago), crashers: 0, restarts: 1/84, execs: 84 (2/sec), cover: 0, uptime: 36s +2016/09/16 13:59:11 slaves: 8, corpus: 2 (0s ago), crashers: 0, restarts: 1/85, execs: 85 (2/sec), cover: 0, uptime: 39s +2016/09/16 13:59:14 slaves: 8, corpus: 17 (2s ago), crashers: 0, restarts: 1/86, execs: 86 (2/sec), cover: 480, uptime: 42s +2016/09/16 13:59:17 slaves: 8, corpus: 17 (5s ago), crashers: 0, restarts: 1/66, execs: 132 (3/sec), cover: 487, uptime: 45s +2016/09/16 13:59:20 slaves: 8, corpus: 17 (8s ago), crashers: 0, restarts: 1/440, execs: 2645 (55/sec), cover: 487, uptime: 48s + +``` + +Let it run, and if the # of crashers is > 0, check out the reports in +the workdir where you should be able to find the panic goroutine stack +traces. +*/ + +package roaring + +import ( + "fmt" + "sort" + + "github.com/mschoch/smat" + "github.com/willf/bitset" +) + +// fuzz test using state machine driven by byte stream. +func FuzzSmat(data []byte) int { + return smat.Fuzz(&smatContext{}, smat.ActionID('S'), smat.ActionID('T'), + smatActionMap, data) +} + +var smatDebug = false + +func smatLog(prefix, format string, args ...interface{}) { + if smatDebug { + fmt.Print(prefix) + fmt.Printf(format, args...) + } +} + +type smatContext struct { + pairs []*smatPair + + // Two registers, x & y. + x int + y int + + actions int +} + +type smatPair struct { + bm *Bitmap + bs *bitset.BitSet +} + +// ------------------------------------------------------------------ + +var smatActionMap = smat.ActionMap{ + smat.ActionID('X'): smatAction("x++", smatWrap(func(c *smatContext) { c.x++ })), + smat.ActionID('x'): smatAction("x--", smatWrap(func(c *smatContext) { c.x-- })), + smat.ActionID('Y'): smatAction("y++", smatWrap(func(c *smatContext) { c.y++ })), + smat.ActionID('y'): smatAction("y--", smatWrap(func(c *smatContext) { c.y-- })), + smat.ActionID('*'): smatAction("x*y", smatWrap(func(c *smatContext) { c.x = c.x * c.y })), + smat.ActionID('<'): smatAction("x<<", smatWrap(func(c *smatContext) { c.x = c.x << 1 })), + + smat.ActionID('^'): smatAction("swap", smatWrap(func(c *smatContext) { c.x, c.y = c.y, c.x })), + + smat.ActionID('['): smatAction(" pushPair", smatWrap(smatPushPair)), + smat.ActionID(']'): smatAction(" popPair", smatWrap(smatPopPair)), + + smat.ActionID('B'): smatAction(" setBit", smatWrap(smatSetBit)), + smat.ActionID('b'): smatAction(" removeBit", smatWrap(smatRemoveBit)), + + smat.ActionID('o'): smatAction(" or", smatWrap(smatOr)), + smat.ActionID('a'): smatAction(" and", smatWrap(smatAnd)), + + smat.ActionID('#'): smatAction(" cardinality", smatWrap(smatCardinality)), + + smat.ActionID('O'): smatAction(" orCardinality", smatWrap(smatOrCardinality)), + smat.ActionID('A'): smatAction(" andCardinality", smatWrap(smatAndCardinality)), + + smat.ActionID('c'): smatAction(" clear", smatWrap(smatClear)), + smat.ActionID('r'): smatAction(" runOptimize", smatWrap(smatRunOptimize)), + + smat.ActionID('e'): smatAction(" isEmpty", smatWrap(smatIsEmpty)), + + smat.ActionID('i'): smatAction(" intersects", smatWrap(smatIntersects)), + + smat.ActionID('f'): smatAction(" flip", smatWrap(smatFlip)), + + smat.ActionID('-'): smatAction(" difference", smatWrap(smatDifference)), +} + +var smatRunningPercentActions []smat.PercentAction + +func init() { + var ids []int + for actionId := range smatActionMap { + ids = append(ids, int(actionId)) + } + sort.Ints(ids) + + pct := 100 / len(smatActionMap) + for _, actionId := range ids { + smatRunningPercentActions = append(smatRunningPercentActions, + smat.PercentAction{pct, smat.ActionID(actionId)}) + } + + smatActionMap[smat.ActionID('S')] = smatAction("SETUP", smatSetupFunc) + smatActionMap[smat.ActionID('T')] = smatAction("TEARDOWN", smatTeardownFunc) +} + +// We only have one smat state: running. +func smatRunning(next byte) smat.ActionID { + return smat.PercentExecute(next, smatRunningPercentActions...) +} + +func smatAction(name string, f func(ctx smat.Context) (smat.State, error)) func(smat.Context) (smat.State, error) { + return func(ctx smat.Context) (smat.State, error) { + c := ctx.(*smatContext) + c.actions++ + + smatLog(" ", "%s\n", name) + + return f(ctx) + } +} + +// Creates an smat action func based on a simple callback. +func smatWrap(cb func(c *smatContext)) func(smat.Context) (next smat.State, err error) { + return func(ctx smat.Context) (next smat.State, err error) { + c := ctx.(*smatContext) + cb(c) + return smatRunning, nil + } +} + +// Invokes a callback function with the input v bounded to len(c.pairs). +func (c *smatContext) withPair(v int, cb func(*smatPair)) { + if len(c.pairs) > 0 { + if v < 0 { + v = -v + } + v = v % len(c.pairs) + cb(c.pairs[v]) + } +} + +// ------------------------------------------------------------------ + +func smatSetupFunc(ctx smat.Context) (next smat.State, err error) { + return smatRunning, nil +} + +func smatTeardownFunc(ctx smat.Context) (next smat.State, err error) { + return nil, err +} + +// ------------------------------------------------------------------ + +func smatPushPair(c *smatContext) { + c.pairs = append(c.pairs, &smatPair{ + bm: NewBitmap(), + bs: bitset.New(100), + }) +} + +func smatPopPair(c *smatContext) { + if len(c.pairs) > 0 { + c.pairs = c.pairs[0 : len(c.pairs)-1] + } +} + +func smatSetBit(c *smatContext) { + c.withPair(c.x, func(p *smatPair) { + y := uint32(c.y) + p.bm.AddInt(int(y)) + p.bs.Set(uint(y)) + p.checkEquals() + }) +} + +func smatRemoveBit(c *smatContext) { + c.withPair(c.x, func(p *smatPair) { + y := uint32(c.y) + p.bm.Remove(y) + p.bs.Clear(uint(y)) + p.checkEquals() + }) +} + +func smatAnd(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + px.bm.And(py.bm) + px.bs = px.bs.Intersection(py.bs) + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatOr(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + px.bm.Or(py.bm) + px.bs = px.bs.Union(py.bs) + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatAndCardinality(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + c0 := px.bm.AndCardinality(py.bm) + c1 := px.bs.IntersectionCardinality(py.bs) + if c0 != uint64(c1) { + panic("expected same add cardinality") + } + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatOrCardinality(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + c0 := px.bm.OrCardinality(py.bm) + c1 := px.bs.UnionCardinality(py.bs) + if c0 != uint64(c1) { + panic("expected same or cardinality") + } + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatRunOptimize(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + px.bm.RunOptimize() + px.checkEquals() + }) +} + +func smatClear(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + px.bm.Clear() + px.bs = px.bs.ClearAll() + px.checkEquals() + }) +} + +func smatCardinality(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c0 := px.bm.GetCardinality() + c1 := px.bs.Count() + if c0 != uint64(c1) { + panic("expected same cardinality") + } + }) +} + +func smatIsEmpty(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c0 := px.bm.IsEmpty() + c1 := px.bs.None() + if c0 != c1 { + panic("expected same is empty") + } + }) +} + +func smatIntersects(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + v0 := px.bm.Intersects(py.bm) + v1 := px.bs.IntersectionCardinality(py.bs) > 0 + if v0 != v1 { + panic("intersects not equal") + } + + px.checkEquals() + py.checkEquals() + }) + }) +} + +func smatFlip(c *smatContext) { + c.withPair(c.x, func(p *smatPair) { + y := uint32(c.y) + p.bm.Flip(uint64(y), uint64(y)+1) + p.bs = p.bs.Flip(uint(y)) + p.checkEquals() + }) +} + +func smatDifference(c *smatContext) { + c.withPair(c.x, func(px *smatPair) { + c.withPair(c.y, func(py *smatPair) { + px.bm.AndNot(py.bm) + px.bs = px.bs.Difference(py.bs) + px.checkEquals() + py.checkEquals() + }) + }) +} + +func (p *smatPair) checkEquals() { + if !p.equalsBitSet(p.bs, p.bm) { + panic("bitset mismatch") + } +} + +func (p *smatPair) equalsBitSet(a *bitset.BitSet, b *Bitmap) bool { + for i, e := a.NextSet(0); e; i, e = a.NextSet(i + 1) { + if !b.ContainsInt(int(i)) { + fmt.Printf("in a bitset, not b bitmap, i: %d\n", i) + fmt.Printf(" a bitset: %s\n b bitmap: %s\n", + a.String(), b.String()) + return false + } + } + + i := b.Iterator() + for i.HasNext() { + v := i.Next() + if !a.Test(uint(v)) { + fmt.Printf("in b bitmap, not a bitset, v: %d\n", v) + fmt.Printf(" a bitset: %s\n b bitmap: %s\n", + a.String(), b.String()) + return false + } + } + + return true +} diff --git a/vendor/github.com/RoaringBitmap/roaring/util.go b/vendor/github.com/RoaringBitmap/roaring/util.go new file mode 100644 index 00000000..d212660d --- /dev/null +++ b/vendor/github.com/RoaringBitmap/roaring/util.go @@ -0,0 +1,315 @@ +package roaring + +import ( + "math/rand" + "sort" +) + +const ( + arrayDefaultMaxSize = 4096 // containers with 4096 or fewer integers should be array containers. + arrayLazyLowerBound = 1024 + maxCapacity = 1 << 16 + serialCookieNoRunContainer = 12346 // only arrays and bitmaps + invalidCardinality = -1 + serialCookie = 12347 // runs, arrays, and bitmaps + noOffsetThreshold = 4 + + // Compute wordSizeInBytes, the size of a word in bytes. + _m = ^uint64(0) + _logS = _m>>8&1 + _m>>16&1 + _m>>32&1 + wordSizeInBytes = 1 << _logS + + // other constants used in ctz_generic.go + wordSizeInBits = wordSizeInBytes << 3 // word size in bits +) + +const maxWord = 1< arrayDefaultMaxSize { + // bitmapContainer + return maxCapacity / 8 + } + // arrayContainer + return 2 * card +} + +func fill(arr []uint64, val uint64) { + for i := range arr { + arr[i] = val + } +} +func fillRange(arr []uint64, start, end int, val uint64) { + for i := start; i < end; i++ { + arr[i] = val + } +} + +func fillArrayAND(container []uint16, bitmap1, bitmap2 []uint64) { + if len(bitmap1) != len(bitmap2) { + panic("array lengths don't match") + } + // TODO: rewrite in assembly + pos := 0 + for k := range bitmap1 { + bitset := bitmap1[k] & bitmap2[k] + for bitset != 0 { + t := bitset & -bitset + container[pos] = uint16((k*64 + int(popcount(t-1)))) + pos = pos + 1 + bitset ^= t + } + } +} + +func fillArrayANDNOT(container []uint16, bitmap1, bitmap2 []uint64) { + if len(bitmap1) != len(bitmap2) { + panic("array lengths don't match") + } + // TODO: rewrite in assembly + pos := 0 + for k := range bitmap1 { + bitset := bitmap1[k] &^ bitmap2[k] + for bitset != 0 { + t := bitset & -bitset + container[pos] = uint16((k*64 + int(popcount(t-1)))) + pos = pos + 1 + bitset ^= t + } + } +} + +func fillArrayXOR(container []uint16, bitmap1, bitmap2 []uint64) { + if len(bitmap1) != len(bitmap2) { + panic("array lengths don't match") + } + // TODO: rewrite in assembly + pos := 0 + for k := 0; k < len(bitmap1); k++ { + bitset := bitmap1[k] ^ bitmap2[k] + for bitset != 0 { + t := bitset & -bitset + container[pos] = uint16((k*64 + int(popcount(t-1)))) + pos = pos + 1 + bitset ^= t + } + } +} + +func highbits(x uint32) uint16 { + return uint16(x >> 16) +} +func lowbits(x uint32) uint16 { + return uint16(x & 0xFFFF) +} + +const maxLowBit = 0xFFFF + +func flipBitmapRange(bitmap []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + bitmap[firstword] ^= ^(^uint64(0) << uint(start%64)) + for i := firstword; i < endword; i++ { + //p("flipBitmapRange on i=%v", i) + bitmap[i] = ^bitmap[i] + } + bitmap[endword] ^= ^uint64(0) >> (uint(-end) % 64) +} + +func resetBitmapRange(bitmap []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + bitmap[firstword] &= ^((^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64))) + return + } + bitmap[firstword] &= ^(^uint64(0) << uint(start%64)) + for i := firstword + 1; i < endword; i++ { + bitmap[i] = 0 + } + bitmap[endword] &= ^(^uint64(0) >> (uint(-end) % 64)) + +} + +func setBitmapRange(bitmap []uint64, start int, end int) { + if start >= end { + return + } + firstword := start / 64 + endword := (end - 1) / 64 + if firstword == endword { + bitmap[firstword] |= (^uint64(0) << uint(start%64)) & (^uint64(0) >> (uint(-end) % 64)) + return + } + bitmap[firstword] |= ^uint64(0) << uint(start%64) + for i := firstword + 1; i < endword; i++ { + bitmap[i] = ^uint64(0) + } + bitmap[endword] |= ^uint64(0) >> (uint(-end) % 64) +} + +func flipBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { + before := wordCardinalityForBitmapRange(bitmap, start, end) + flipBitmapRange(bitmap, start, end) + after := wordCardinalityForBitmapRange(bitmap, start, end) + return int(after - before) +} + +func resetBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { + before := wordCardinalityForBitmapRange(bitmap, start, end) + resetBitmapRange(bitmap, start, end) + after := wordCardinalityForBitmapRange(bitmap, start, end) + return int(after - before) +} + +func setBitmapRangeAndCardinalityChange(bitmap []uint64, start int, end int) int { + before := wordCardinalityForBitmapRange(bitmap, start, end) + setBitmapRange(bitmap, start, end) + after := wordCardinalityForBitmapRange(bitmap, start, end) + return int(after - before) +} + +func wordCardinalityForBitmapRange(bitmap []uint64, start int, end int) uint64 { + answer := uint64(0) + if start >= end { + return answer + } + firstword := start / 64 + endword := (end - 1) / 64 + for i := firstword; i <= endword; i++ { + answer += popcount(bitmap[i]) + } + return answer +} + +func selectBitPosition(w uint64, j int) int { + seen := 0 + + // Divide 64bit + part := w & 0xFFFFFFFF + n := popcount(part) + if n <= uint64(j) { + part = w >> 32 + seen += 32 + j -= int(n) + } + w = part + + // Divide 32bit + part = w & 0xFFFF + n = popcount(part) + if n <= uint64(j) { + part = w >> 16 + seen += 16 + j -= int(n) + } + w = part + + // Divide 16bit + part = w & 0xFF + n = popcount(part) + if n <= uint64(j) { + part = w >> 8 + seen += 8 + j -= int(n) + } + w = part + + // Lookup in final byte + var counter uint + for counter = 0; counter < 8; counter++ { + j -= int((w >> counter) & 1) + if j < 0 { + break + } + } + return seen + int(counter) + +} + +func panicOn(err error) { + if err != nil { + panic(err) + } +} + +type ph struct { + orig int + rand int +} + +type pha []ph + +func (p pha) Len() int { return len(p) } +func (p pha) Less(i, j int) bool { return p[i].rand < p[j].rand } +func (p pha) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func getRandomPermutation(n int) []int { + r := make([]ph, n) + for i := 0; i < n; i++ { + r[i].orig = i + r[i].rand = rand.Intn(1 << 29) + } + sort.Sort(pha(r)) + m := make([]int, n) + for i := range m { + m[i] = r[i].orig + } + return m +} + +func minOfInt(a, b int) int { + if a < b { + return a + } + return b +} + +func maxOfInt(a, b int) int { + if a > b { + return a + } + return b +} + +func maxOfUint16(a, b uint16) uint16 { + if a > b { + return a + } + return b +} + +func minOfUint16(a, b uint16) uint16 { + if a < b { + return a + } + return b +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} + +func maxUint16(a, b uint16) uint16 { + if a > b { + return a + } + return b +} + +func minUint16(a, b uint16) uint16 { + if a < b { + return a + } + return b +} diff --git a/vendor/github.com/blevesearch/bleve/README.md b/vendor/github.com/blevesearch/bleve/README.md index fa11f906..7c1a7c7c 100644 --- a/vendor/github.com/blevesearch/bleve/README.md +++ b/vendor/github.com/blevesearch/bleve/README.md @@ -1,6 +1,6 @@ # ![bleve](docs/bleve.png) bleve -[![Build Status](https://travis-ci.org/blevesearch/bleve.svg?branch=master)](https://travis-ci.org/blevesearch/bleve) [![Coverage Status](https://coveralls.io/repos/blevesearch/bleve/badge.png?branch=master)](https://coveralls.io/r/blevesearch/bleve?branch=master) [![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve) +[![Build Status](https://travis-ci.org/blevesearch/bleve.svg?branch=master)](https://travis-ci.org/blevesearch/bleve) [![Coverage Status](https://coveralls.io/repos/github/blevesearch/bleve/badge.svg?branch=master)](https://coveralls.io/github/blevesearch/bleve?branch=master) [![GoDoc](https://godoc.org/github.com/blevesearch/bleve?status.svg)](https://godoc.org/github.com/blevesearch/bleve) [![Join the chat at https://gitter.im/blevesearch/bleve](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/blevesearch/bleve?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![codebeat](https://codebeat.co/badges/38a7cbc9-9cf5-41c0-a315-0746178230f4)](https://codebeat.co/projects/github-com-blevesearch-bleve) [![Go Report Card](https://goreportcard.com/badge/blevesearch/bleve)](https://goreportcard.com/report/blevesearch/bleve) diff --git a/vendor/github.com/blevesearch/bleve/analysis/freq.go b/vendor/github.com/blevesearch/bleve/analysis/freq.go index e1ca2cd6..198c149b 100644 --- a/vendor/github.com/blevesearch/bleve/analysis/freq.go +++ b/vendor/github.com/blevesearch/bleve/analysis/freq.go @@ -14,6 +14,22 @@ package analysis +import ( + "reflect" + + "github.com/blevesearch/bleve/size" +) + +var reflectStaticSizeTokenLocation int +var reflectStaticSizeTokenFreq int + +func init() { + var tl TokenLocation + reflectStaticSizeTokenLocation = int(reflect.TypeOf(tl).Size()) + var tf TokenFreq + reflectStaticSizeTokenFreq = int(reflect.TypeOf(tf).Size()) +} + // TokenLocation represents one occurrence of a term at a particular location in // a field. Start, End and Position have the same meaning as in analysis.Token. // Field and ArrayPositions identify the field value in the source document. @@ -26,6 +42,12 @@ type TokenLocation struct { Position int } +func (tl *TokenLocation) Size() int { + rv := reflectStaticSizeTokenLocation + rv += len(tl.ArrayPositions) * size.SizeOfUint64 + return rv +} + // TokenFreq represents all the occurrences of a term in all fields of a // document. type TokenFreq struct { @@ -34,6 +56,15 @@ type TokenFreq struct { frequency int } +func (tf *TokenFreq) Size() int { + rv := reflectStaticSizeTokenFreq + rv += len(tf.Term) + for _, loc := range tf.Locations { + rv += loc.Size() + } + return rv +} + func (tf *TokenFreq) Frequency() int { return tf.frequency } @@ -42,6 +73,16 @@ func (tf *TokenFreq) Frequency() int { // fields. type TokenFrequencies map[string]*TokenFreq +func (tfs TokenFrequencies) Size() int { + rv := size.SizeOfMap + rv += len(tfs) * (size.SizeOfString + size.SizeOfPtr) + for k, v := range tfs { + rv += len(k) + rv += v.Size() + } + return rv +} + func (tfs TokenFrequencies) MergeAll(remoteField string, other TokenFrequencies) { // walk the new token frequencies for tfk, tf := range other { diff --git a/vendor/github.com/blevesearch/bleve/config.go b/vendor/github.com/blevesearch/bleve/config.go index 74d407fd..482efb40 100644 --- a/vendor/github.com/blevesearch/bleve/config.go +++ b/vendor/github.com/blevesearch/bleve/config.go @@ -25,6 +25,9 @@ import ( "github.com/blevesearch/bleve/index/upsidedown" "github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/search/highlight/highlighter/html" + + // force import of scorch so its accessible by default + _ "github.com/blevesearch/bleve/index/scorch" ) var bleveExpVar = expvar.NewMap("bleve") diff --git a/vendor/github.com/blevesearch/bleve/document/document.go b/vendor/github.com/blevesearch/bleve/document/document.go index ed36b127..6ac17b9a 100644 --- a/vendor/github.com/blevesearch/bleve/document/document.go +++ b/vendor/github.com/blevesearch/bleve/document/document.go @@ -14,13 +14,24 @@ package document -import "fmt" +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/size" +) + +var reflectStaticSizeDocument int + +func init() { + var d Document + reflectStaticSizeDocument = int(reflect.TypeOf(d).Size()) +} type Document struct { ID string `json:"id"` Fields []Field `json:"fields"` CompositeFields []*CompositeField - Number uint64 `json:"-"` } func NewDocument(id string) *Document { @@ -31,6 +42,21 @@ func NewDocument(id string) *Document { } } +func (d *Document) Size() int { + sizeInBytes := reflectStaticSizeDocument + size.SizeOfPtr + + len(d.ID) + + for _, entry := range d.Fields { + sizeInBytes += entry.Size() + } + + for _, entry := range d.CompositeFields { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + func (d *Document) AddField(f Field) *Document { switch f := f.(type) { case *CompositeField: diff --git a/vendor/github.com/blevesearch/bleve/document/field.go b/vendor/github.com/blevesearch/bleve/document/field.go index c17f81e5..2fe91669 100644 --- a/vendor/github.com/blevesearch/bleve/document/field.go +++ b/vendor/github.com/blevesearch/bleve/document/field.go @@ -36,4 +36,6 @@ type Field interface { // that this field represents - this is a common metric for tracking // the rate of indexing NumPlainTextBytes() uint64 + + Size() int } diff --git a/vendor/github.com/blevesearch/bleve/document/field_boolean.go b/vendor/github.com/blevesearch/bleve/document/field_boolean.go index 668b431a..6864b16f 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_boolean.go +++ b/vendor/github.com/blevesearch/bleve/document/field_boolean.go @@ -16,11 +16,20 @@ package document import ( "fmt" + "reflect" "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/size" ) -const DefaultBooleanIndexingOptions = StoreField | IndexField +var reflectStaticSizeBooleanField int + +func init() { + var f BooleanField + reflectStaticSizeBooleanField = int(reflect.TypeOf(f).Size()) +} + +const DefaultBooleanIndexingOptions = StoreField | IndexField | DocValues type BooleanField struct { name string @@ -30,6 +39,13 @@ type BooleanField struct { numPlainTextBytes uint64 } +func (b *BooleanField) Size() int { + return reflectStaticSizeBooleanField + size.SizeOfPtr + + len(b.name) + + len(b.arrayPositions)*size.SizeOfUint64 + + len(b.value) +} + func (b *BooleanField) Name() string { return b.name } diff --git a/vendor/github.com/blevesearch/bleve/document/field_composite.go b/vendor/github.com/blevesearch/bleve/document/field_composite.go index b41b1b8e..a8285880 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_composite.go +++ b/vendor/github.com/blevesearch/bleve/document/field_composite.go @@ -15,9 +15,19 @@ package document import ( + "reflect" + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeCompositeField int + +func init() { + var cf CompositeField + reflectStaticSizeCompositeField = int(reflect.TypeOf(cf).Size()) +} + const DefaultCompositeIndexingOptions = IndexField type CompositeField struct { @@ -54,6 +64,21 @@ func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, incl return rv } +func (c *CompositeField) Size() int { + sizeInBytes := reflectStaticSizeCompositeField + size.SizeOfPtr + + len(c.name) + + for k, _ := range c.includedFields { + sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool + } + + for k, _ := range c.excludedFields { + sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool + } + + return sizeInBytes +} + func (c *CompositeField) Name() string { return c.name } diff --git a/vendor/github.com/blevesearch/bleve/document/field_datetime.go b/vendor/github.com/blevesearch/bleve/document/field_datetime.go index 6783d53d..583b44cd 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_datetime.go +++ b/vendor/github.com/blevesearch/bleve/document/field_datetime.go @@ -17,13 +17,22 @@ package document import ( "fmt" "math" + "reflect" "time" "github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/size" ) -const DefaultDateTimeIndexingOptions = StoreField | IndexField +var reflectStaticSizeDateTimeField int + +func init() { + var f DateTimeField + reflectStaticSizeDateTimeField = int(reflect.TypeOf(f).Size()) +} + +const DefaultDateTimeIndexingOptions = StoreField | IndexField | DocValues const DefaultDateTimePrecisionStep uint = 4 var MinTimeRepresentable = time.Unix(0, math.MinInt64) @@ -37,6 +46,12 @@ type DateTimeField struct { numPlainTextBytes uint64 } +func (n *DateTimeField) Size() int { + return reflectStaticSizeDateTimeField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 +} + func (n *DateTimeField) Name() string { return n.name } diff --git a/vendor/github.com/blevesearch/bleve/document/field_geopoint.go b/vendor/github.com/blevesearch/bleve/document/field_geopoint.go index f508b362..91fe23f9 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_geopoint.go +++ b/vendor/github.com/blevesearch/bleve/document/field_geopoint.go @@ -16,12 +16,21 @@ package document import ( "fmt" + "reflect" "github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/geo" "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeGeoPointField int + +func init() { + var f GeoPointField + reflectStaticSizeGeoPointField = int(reflect.TypeOf(f).Size()) +} + var GeoPrecisionStep uint = 9 type GeoPointField struct { @@ -32,6 +41,12 @@ type GeoPointField struct { numPlainTextBytes uint64 } +func (n *GeoPointField) Size() int { + return reflectStaticSizeGeoPointField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfUint64 +} + func (n *GeoPointField) Name() string { return n.name } diff --git a/vendor/github.com/blevesearch/bleve/document/field_numeric.go b/vendor/github.com/blevesearch/bleve/document/field_numeric.go index 7faae2bb..46c685e8 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_numeric.go +++ b/vendor/github.com/blevesearch/bleve/document/field_numeric.go @@ -16,12 +16,21 @@ package document import ( "fmt" + "reflect" "github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/numeric" + "github.com/blevesearch/bleve/size" ) -const DefaultNumericIndexingOptions = StoreField | IndexField +var reflectStaticSizeNumericField int + +func init() { + var f NumericField + reflectStaticSizeNumericField = int(reflect.TypeOf(f).Size()) +} + +const DefaultNumericIndexingOptions = StoreField | IndexField | DocValues const DefaultPrecisionStep uint = 4 @@ -33,6 +42,12 @@ type NumericField struct { numPlainTextBytes uint64 } +func (n *NumericField) Size() int { + return reflectStaticSizeNumericField + size.SizeOfPtr + + len(n.name) + + len(n.arrayPositions)*size.SizeOfPtr +} + func (n *NumericField) Name() string { return n.name } diff --git a/vendor/github.com/blevesearch/bleve/document/field_text.go b/vendor/github.com/blevesearch/bleve/document/field_text.go index 37873d36..c8e871c9 100644 --- a/vendor/github.com/blevesearch/bleve/document/field_text.go +++ b/vendor/github.com/blevesearch/bleve/document/field_text.go @@ -16,11 +16,20 @@ package document import ( "fmt" + "reflect" "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/size" ) -const DefaultTextIndexingOptions = IndexField +var reflectStaticSizeTextField int + +func init() { + var f TextField + reflectStaticSizeTextField = int(reflect.TypeOf(f).Size()) +} + +const DefaultTextIndexingOptions = IndexField | DocValues type TextField struct { name string @@ -31,6 +40,13 @@ type TextField struct { numPlainTextBytes uint64 } +func (t *TextField) Size() int { + return reflectStaticSizeTextField + size.SizeOfPtr + + len(t.name) + + len(t.arrayPositions)*size.SizeOfUint64 + + len(t.value) +} + func (t *TextField) Name() string { return t.name } diff --git a/vendor/github.com/blevesearch/bleve/document/indexing_options.go b/vendor/github.com/blevesearch/bleve/document/indexing_options.go index 5d562c1d..44498a8e 100644 --- a/vendor/github.com/blevesearch/bleve/document/indexing_options.go +++ b/vendor/github.com/blevesearch/bleve/document/indexing_options.go @@ -20,6 +20,7 @@ const ( IndexField IndexingOptions = 1 << iota StoreField IncludeTermVectors + DocValues ) func (o IndexingOptions) IsIndexed() bool { @@ -34,6 +35,10 @@ func (o IndexingOptions) IncludeTermVectors() bool { return o&IncludeTermVectors != 0 } +func (o IndexingOptions) IncludeDocValues() bool { + return o&DocValues != 0 +} + func (o IndexingOptions) String() string { rv := "" if o.IsIndexed() { @@ -51,5 +56,11 @@ func (o IndexingOptions) String() string { } rv += "TV" } + if o.IncludeDocValues() { + if rv != "" { + rv += ", " + } + rv += "DV" + } return rv } diff --git a/vendor/github.com/blevesearch/bleve/geo/geo.go b/vendor/github.com/blevesearch/bleve/geo/geo.go index 3212539b..86861b4f 100644 --- a/vendor/github.com/blevesearch/bleve/geo/geo.go +++ b/vendor/github.com/blevesearch/bleve/geo/geo.go @@ -15,6 +15,7 @@ package geo import ( + "fmt" "math" "github.com/blevesearch/bleve/numeric" @@ -26,6 +27,12 @@ var GeoBits uint = 32 var minLon = -180.0 var minLat = -90.0 +var maxLon = 180.0 +var maxLat = 90.0 +var minLonRad = minLon * degreesToRadian +var minLatRad = minLat * degreesToRadian +var maxLonRad = maxLon * degreesToRadian +var maxLatRad = maxLat * degreesToRadian var geoTolerance = 1E-6 var lonScale = float64((uint64(0x1)<= 0 && compareGeo(lat, maxLat) <= 0 } -// ComputeBoundingBox will compute a bounding box around the provided point -// which surrounds a circle of the provided radius (in meters). -func ComputeBoundingBox(centerLon, centerLat, - radius float64) (upperLeftLon float64, upperLeftLat float64, - lowerRightLon float64, lowerRightLat float64) { - _, tlat := pointFromLonLatBearing(centerLon, centerLat, 0, radius) - rlon, _ := pointFromLonLatBearing(centerLon, centerLat, 90, radius) - _, blat := pointFromLonLatBearing(centerLon, centerLat, 180, radius) - llon, _ := pointFromLonLatBearing(centerLon, centerLat, 270, radius) - return normalizeLon(llon), normalizeLat(tlat), - normalizeLon(rlon), normalizeLat(blat) -} - const degreesToRadian = math.Pi / 180 const radiansToDegrees = 180 / math.Pi -const flattening = 1.0 / 298.257223563 -const semiMajorAxis = 6378137 -const semiMinorAxis = semiMajorAxis * (1.0 - flattening) -const semiMajorAxis2 = semiMajorAxis * semiMajorAxis -const semiMinorAxis2 = semiMinorAxis * semiMinorAxis // DegreesToRadians converts an angle in degrees to radians func DegreesToRadians(d float64) float64 { @@ -122,84 +111,60 @@ func RadiansToDegrees(r float64) float64 { return r * radiansToDegrees } -// pointFromLonLatBearing starts that the provide lon,lat -// then moves in the bearing direction (in degrees) -// this move continues for the provided distance (in meters) -// The lon, lat of this destination location is returned. -func pointFromLonLatBearing(lon, lat, bearing, - dist float64) (float64, float64) { +var earthMeanRadiusMeters = 6371008.7714 - alpha1 := DegreesToRadians(bearing) - cosA1 := math.Cos(alpha1) - sinA1 := math.Sin(alpha1) - tanU1 := (1 - flattening) * math.Tan(DegreesToRadians(lat)) - cosU1 := 1 / math.Sqrt(1+tanU1*tanU1) - sinU1 := tanU1 * cosU1 - sig1 := math.Atan2(tanU1, cosA1) - sinAlpha := cosU1 * sinA1 - cosSqAlpha := 1 - sinAlpha*sinAlpha - uSq := cosSqAlpha * (semiMajorAxis2 - semiMinorAxis2) / semiMinorAxis2 - A := 1 + uSq/16384*(4096+uSq*(-768+uSq*(320-175*uSq))) - B := uSq / 1024 * (256 + uSq*(-128+uSq*(74-47*uSq))) +func RectFromPointDistance(lon, lat, dist float64) (float64, float64, float64, float64, error) { + err := checkLongitude(lon) + if err != nil { + return 0, 0, 0, 0, err + } + err = checkLatitude(lat) + if err != nil { + return 0, 0, 0, 0, err + } + radLon := DegreesToRadians(lon) + radLat := DegreesToRadians(lat) + radDistance := (dist + 7e-2) / earthMeanRadiusMeters - sigma := dist / (semiMinorAxis * A) + minLatL := radLat - radDistance + maxLatL := radLat + radDistance - cos25SigmaM := math.Cos(2*sig1 + sigma) - sinSigma := math.Sin(sigma) - cosSigma := math.Cos(sigma) - deltaSigma := B * sinSigma * (cos25SigmaM + (B/4)* - (cosSigma*(-1+2*cos25SigmaM*cos25SigmaM)-(B/6)*cos25SigmaM* - (-1+4*sinSigma*sinSigma)*(-3+4*cos25SigmaM*cos25SigmaM))) - sigmaP := sigma - sigma = dist/(semiMinorAxis*A) + deltaSigma - for math.Abs(sigma-sigmaP) > 1E-12 { - cos25SigmaM = math.Cos(2*sig1 + sigma) - sinSigma = math.Sin(sigma) - cosSigma = math.Cos(sigma) - deltaSigma = B * sinSigma * (cos25SigmaM + (B/4)* - (cosSigma*(-1+2*cos25SigmaM*cos25SigmaM)-(B/6)*cos25SigmaM* - (-1+4*sinSigma*sinSigma)*(-3+4*cos25SigmaM*cos25SigmaM))) - sigmaP = sigma - sigma = dist/(semiMinorAxis*A) + deltaSigma + var minLonL, maxLonL float64 + if minLatL > minLatRad && maxLatL < maxLatRad { + deltaLon := asin(sin(radDistance) / cos(radLat)) + minLonL = radLon - deltaLon + if minLonL < minLonRad { + minLonL += 2 * math.Pi + } + maxLonL = radLon + deltaLon + if maxLonL > maxLonRad { + maxLonL -= 2 * math.Pi + } + } else { + // pole is inside distance + minLatL = math.Max(minLatL, minLatRad) + maxLatL = math.Min(maxLatL, maxLatRad) + minLonL = minLonRad + maxLonL = maxLonRad } - tmp := sinU1*sinSigma - cosU1*cosSigma*cosA1 - lat2 := math.Atan2(sinU1*cosSigma+cosU1*sinSigma*cosA1, - (1-flattening)*math.Sqrt(sinAlpha*sinAlpha+tmp*tmp)) - lamda := math.Atan2(sinSigma*sinA1, cosU1*cosSigma-sinU1*sinSigma*cosA1) - c := flattening / 16 * cosSqAlpha * (4 + flattening*(4-3*cosSqAlpha)) - lam := lamda - (1-c)*flattening*sinAlpha* - (sigma+c*sinSigma*(cos25SigmaM+c*cosSigma*(-1+2*cos25SigmaM*cos25SigmaM))) - - rvlon := lon + RadiansToDegrees(lam) - rvlat := RadiansToDegrees(lat2) - - return rvlon, rvlat + return RadiansToDegrees(minLonL), + RadiansToDegrees(maxLatL), + RadiansToDegrees(maxLonL), + RadiansToDegrees(minLatL), + nil } -// normalizeLon normalizes a longitude value within the -180 to 180 range -func normalizeLon(lonDeg float64) float64 { - if lonDeg >= -180 && lonDeg <= 180 { - return lonDeg +func checkLatitude(latitude float64) error { + if math.IsNaN(latitude) || latitude < minLat || latitude > maxLat { + return fmt.Errorf("invalid latitude %f; must be between %f and %f", latitude, minLat, maxLat) } - - off := math.Mod(lonDeg+180, 360) - if off < 0 { - return 180 + off - } else if off == 0 && lonDeg > 0 { - return 180 - } - return -180 + off + return nil } -// normalizeLat normalizes a latitude value within the -90 to 90 range -func normalizeLat(latDeg float64) float64 { - if latDeg >= -90 && latDeg <= 90 { - return latDeg +func checkLongitude(longitude float64) error { + if math.IsNaN(longitude) || longitude < minLon || longitude > maxLon { + return fmt.Errorf("invalid longitude %f; must be between %f and %f", longitude, minLon, maxLon) } - off := math.Abs(math.Mod(latDeg+90, 360)) - if off <= 180 { - return off - 90 - } - return (360 - off) - 90 + return nil } diff --git a/vendor/github.com/blevesearch/bleve/geo/sloppy.go b/vendor/github.com/blevesearch/bleve/geo/sloppy.go index a0f5a366..0ce646d7 100644 --- a/vendor/github.com/blevesearch/bleve/geo/sloppy.go +++ b/vendor/github.com/blevesearch/bleve/geo/sloppy.go @@ -146,6 +146,12 @@ func earthDiameter(lat float64) float64 { return earthDiameterPerLatitude[int(index)] } +var pio2 = math.Pi / 2 + +func sin(a float64) float64 { + return cos(a - pio2) +} + // cos is a sloppy math (faster) implementation of math.Cos func cos(a float64) float64 { if a < 0.0 { diff --git a/vendor/github.com/blevesearch/bleve/index.go b/vendor/github.com/blevesearch/bleve/index.go index 0fadc573..197f4d4d 100644 --- a/vendor/github.com/blevesearch/bleve/index.go +++ b/vendor/github.com/blevesearch/bleve/index.go @@ -15,11 +15,13 @@ package bleve import ( + "context" + "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" "github.com/blevesearch/bleve/mapping" - "golang.org/x/net/context" + "github.com/blevesearch/bleve/size" ) // A Batch groups together multiple Index and Delete @@ -31,6 +33,9 @@ import ( type Batch struct { index Index internal *index.Batch + + lastDocSize uint64 + totalSize uint64 } // Index adds the specified index operation to the @@ -46,6 +51,30 @@ func (b *Batch) Index(id string, data interface{}) error { return err } b.internal.Update(doc) + + b.lastDocSize = uint64(doc.Size() + + len(id) + size.SizeOfString) // overhead from internal + b.totalSize += b.lastDocSize + + return nil +} + +func (b *Batch) LastDocSize() uint64 { + return b.lastDocSize +} + +func (b *Batch) TotalDocsSize() uint64 { + return b.totalSize +} + +// IndexAdvanced adds the specified index operation to the +// batch which skips the mapping. NOTE: the bleve Index is not updated +// until the batch is executed. +func (b *Batch) IndexAdvanced(doc *document.Document) (err error) { + if doc.ID == "" { + return ErrorEmptyID + } + b.internal.Update(doc) return nil } @@ -65,7 +94,7 @@ func (b *Batch) SetInternal(key, val []byte) { b.internal.SetInternal(key, val) } -// SetInternal adds the specified delete internal +// DeleteInternal adds the specified delete internal // operation to the batch. NOTE: the bleve Index is // not updated until the batch is executed. func (b *Batch) DeleteInternal(key []byte) { @@ -99,12 +128,15 @@ func (b *Batch) Reset() { // them. // // The DocumentMapping used to index a value is deduced by the following rules: -// 1) If value implements Classifier interface, resolve the mapping from Type(). -// 2) If value has a string field or value at IndexMapping.TypeField. +// 1) If value implements mapping.bleveClassifier interface, resolve the mapping +// from BleveType(). +// 2) If value implements mapping.Classifier interface, resolve the mapping +// from Type(). +// 3) If value has a string field or value at IndexMapping.TypeField. // (defaulting to "_type"), use it to resolve the mapping. Fields addressing // is described below. -// 3) If IndexMapping.DefaultType is registered, return it. -// 4) Return IndexMapping.DefaultMapping. +// 4) If IndexMapping.DefaultType is registered, return it. +// 5) Return IndexMapping.DefaultMapping. // // Each field or nested field of the value is identified by a string path, then // mapped to one or several FieldMappings which extract the result for analysis. diff --git a/vendor/github.com/blevesearch/bleve/index/analysis.go b/vendor/github.com/blevesearch/bleve/index/analysis.go index b626b9f3..82883af0 100644 --- a/vendor/github.com/blevesearch/bleve/index/analysis.go +++ b/vendor/github.com/blevesearch/bleve/index/analysis.go @@ -14,7 +14,20 @@ package index -import "github.com/blevesearch/bleve/document" +import ( + "reflect" + + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/size" +) + +var reflectStaticSizeAnalysisResult int + +func init() { + var ar AnalysisResult + reflectStaticSizeAnalysisResult = int(reflect.TypeOf(ar).Size()) +} type IndexRow interface { KeySize() int @@ -29,6 +42,20 @@ type IndexRow interface { type AnalysisResult struct { DocID string Rows []IndexRow + + // scorch + Document *document.Document + Analyzed []analysis.TokenFrequencies + Length []int +} + +func (a *AnalysisResult) Size() int { + rv := reflectStaticSizeAnalysisResult + for _, analyzedI := range a.Analyzed { + rv += analyzedI.Size() + } + rv += len(a.Length) * size.SizeOfInt + return rv } type AnalysisWork struct { diff --git a/vendor/github.com/blevesearch/bleve/index/index.go b/vendor/github.com/blevesearch/bleve/index/index.go index 9870b417..7c4edcb8 100644 --- a/vendor/github.com/blevesearch/bleve/index/index.go +++ b/vendor/github.com/blevesearch/bleve/index/index.go @@ -18,11 +18,23 @@ import ( "bytes" "encoding/json" "fmt" + "reflect" "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeTermFieldDoc int +var reflectStaticSizeTermFieldVector int + +func init() { + var tfd TermFieldDoc + reflectStaticSizeTermFieldDoc = int(reflect.TypeOf(tfd).Size()) + var tfv TermFieldVector + reflectStaticSizeTermFieldVector = int(reflect.TypeOf(tfv).Size()) +} + var ErrorUnknownStorageType = fmt.Errorf("unknown storage type") type Index interface { @@ -68,6 +80,8 @@ type IndexReader interface { Document(id string) (*document.Document, error) DocumentVisitFieldTerms(id IndexInternalID, fields []string, visitor DocumentFieldTermVisitor) error + DocValueReader(fields []string) (DocValueReader, error) + Fields() ([]string, error) GetInternal(key []byte) ([]byte, error) @@ -84,6 +98,18 @@ type IndexReader interface { Close() error } +type IndexReaderRegexp interface { + FieldDictRegexp(field string, regex []byte) (FieldDict, error) +} + +type IndexReaderFuzzy interface { + FieldDictFuzzy(field string, term []byte, fuzziness int) (FieldDict, error) +} + +type IndexReaderOnly interface { + FieldDictOnly(field string, onlyTerms [][]byte, includeCount bool) (FieldDict, error) +} + // FieldTerms contains the terms used by a document, keyed by field type FieldTerms map[string][]string @@ -115,6 +141,11 @@ type TermFieldVector struct { End uint64 } +func (tfv *TermFieldVector) Size() int { + return reflectStaticSizeTermFieldVector + size.SizeOfPtr + + len(tfv.Field) + len(tfv.ArrayPositions)*size.SizeOfUint64 +} + // IndexInternalID is an opaque document identifier interal to the index impl type IndexInternalID []byte @@ -134,14 +165,27 @@ type TermFieldDoc struct { Vectors []*TermFieldVector } +func (tfd *TermFieldDoc) Size() int { + sizeInBytes := reflectStaticSizeTermFieldDoc + size.SizeOfPtr + + len(tfd.Term) + len(tfd.ID) + + for _, entry := range tfd.Vectors { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + // Reset allows an already allocated TermFieldDoc to be reused func (tfd *TermFieldDoc) Reset() *TermFieldDoc { // remember the []byte used for the ID id := tfd.ID + vectors := tfd.Vectors // idiom to copy over from empty TermFieldDoc (0 allocations) *tfd = TermFieldDoc{} // reuse the []byte already allocated (and reset len to 0) tfd.ID = id[:0] + tfd.Vectors = vectors[:0] return tfd } @@ -161,6 +205,8 @@ type TermFieldReader interface { // Count returns the number of documents contains the term in this field. Count() uint64 Close() error + + Size() int } type DictEntry struct { @@ -185,6 +231,9 @@ type DocIDReader interface { // will start there instead. If ID is greater than or equal to the end of // the range, Next() call will return io.EOF. Advance(ID IndexInternalID) (IndexInternalID, error) + + Size() int + Close() error } @@ -239,3 +288,23 @@ func (b *Batch) Reset() { b.IndexOps = make(map[string]*document.Document) b.InternalOps = make(map[string][]byte) } + +// Optimizable represents an optional interface that implementable by +// optimizable resources (e.g., TermFieldReaders, Searchers). These +// optimizable resources are provided the same OptimizableContext +// instance, so that they can coordinate via dynamic interface +// casting. +type Optimizable interface { + Optimize(kind string, octx OptimizableContext) (OptimizableContext, error) +} + +type OptimizableContext interface { + // Once all the optimzable resources have been provided the same + // OptimizableContext instance, the optimization preparations are + // finished or completed via the Finish() method. + Finish() error +} + +type DocValueReader interface { + VisitDocValues(id IndexInternalID, visitor DocumentFieldTermVisitor) error +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/README.md b/vendor/github.com/blevesearch/bleve/index/scorch/README.md new file mode 100644 index 00000000..861335a1 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/README.md @@ -0,0 +1,367 @@ +# scorch + +## Definitions + +Batch +- A collection of Documents to mutate in the index. + +Document +- Has a unique identifier (arbitrary bytes). +- Is comprised of a list of fields. + +Field +- Has a name (string). +- Has a type (text, number, date, geopoint). +- Has a value (depending on type). +- Can be indexed, stored, or both. +- If indexed, can be analyzed. +-m If indexed, can optionally store term vectors. + +## Scope + +Scorch *MUST* implement the bleve.index API without requiring any changes to this API. + +Scorch *MAY* introduce new interfaces, which can be discovered to allow use of new capabilities not in the current API. + +## Implementation + +The scorch implementation starts with the concept of a segmented index. + +A segment is simply a slice, subset, or portion of the entire index. A segmented index is one which is composed of one or more segments. Although segments are created in a particular order, knowing this ordering is not required to achieve correct semantics when querying. Because there is no ordering, this means that when searching an index, you can (and should) search all the segments concurrently. + +### Internal Wrapper + +In order to accommodate the existing APIs while also improving the implementation, the scorch implementation includes some wrapper functionality that must be described. + +#### \_id field + +In scorch, field 0 is prearranged to be named \_id. All documents have a value for this field, which is the documents external identifier. In this version the field *MUST* be both indexed AND stored. The scorch wrapper adds this field, as it will not be present in the Document from the calling bleve code. + +NOTE: If a document already contains a field \_id, it will be replaced. If this is problematic, the caller must ensure such a scenario does not happen. + +### Proposed Structures + +``` +type Segment interface { + + Dictionary(field string) TermDictionary + +} + +type TermDictionary interface { + + PostingsList(term string, excluding PostingsList) PostingsList + +} + +type PostingsList interface { + + Next() Posting + + And(other PostingsList) PostingsList + Or(other PostingsList) PostingsList + +} + +type Posting interface { + Number() uint64 + + Frequency() uint64 + Norm() float64 + + Locations() Locations +} + +type Locations interface { + Start() uint64 + End() uint64 + Pos() uint64 + ArrayPositions() ... +} + +type DeletedDocs { + +} + +type SegmentSnapshot struct { + segment Segment + deleted PostingsList +} + +type IndexSnapshot struct { + segment []SegmentSnapshot +} +``` +**What about errors?** +**What about memory mgmnt or context?** +**Postings List separate iterator to separate stateful from stateless** +### Mutating the Index + +The bleve.index API has methods for directly making individual mutations (Update/Delete/SetInternal/DeleteInternal), however for this first implementation, we assume that all of these calls can simply be turned into a Batch of size 1. This may be highly inefficient, but it will be correct. This decision is made based on the fact that Couchbase FTS always uses Batches. + +NOTE: As a side-effect of this decision, it should be clear that performance tuning may depend on the batch size, which may in-turn require changes in FTS. + +From this point forward, only Batch mutations will be discussed. + +Sequence of Operations: + +1. For each document in the batch, search through all existing segments. The goal is to build up a per-segment bitset which tells us which documents in that segment are obsoleted by the addition of the new segment we're currently building. NOTE: we're not ready for this change to take effect yet, so rather than this operation mutating anything, they simply return bitsets, which we can apply later. Logically, this is something like: + + ``` + foreach segment { + dict := segment.Dictionary("\_id") + postings := empty postings list + foreach docID { + postings = postings.Or(dict.PostingsList(docID, nil)) + } + } + ``` + + NOTE: it is illustrated above as nested for loops, but some or all of these could be concurrently. The end result is that for each segment, we have (possibly empty) bitset. + +2. Also concurrent with 1, the documents in the batch are analyzed. This analysis proceeds using the existing analyzer pool. + +3. (after 2 completes) Analyzed documents are fed into a function which builds a new Segment representing this information. + +4. We now have everything we need to update the state of the system to include this new snapshot. + + - Acquire a lock + - Create a new IndexSnapshot + - For each SegmentSnapshot in the IndexSnapshot, take the deleted PostingsList and OR it with the new postings list for this Segment. Construct a new SegmentSnapshot for the segment using this new deleted PostingsList. Append this SegmentSnapshot to the IndexSnapshot. + - Create a new SegmentSnapshot wrapping our new segment with nil deleted docs. + - Append the new SegmentSnapshot to the IndexSnapshot + - Release the lock + +An ASCII art example: + ``` + 0 - Empty Index + + No segments + + IndexSnapshot + segments [] + deleted [] + + + 1 - Index Batch [ A B C ] + + segment 0 + numbers [ 1 2 3 ] + \_id [ A B C ] + + IndexSnapshot + segments [ 0 ] + deleted [ nil ] + + + 2 - Index Batch [ B' ] + + segment 0 1 + numbers [ 1 2 3 ] [ 1 ] + \_id [ A B C ] [ B ] + + Compute bitset segment-0-deleted-by-1: + [ 0 1 0 ] + + OR it with previous (nil) (call it 0-1) + [ 0 1 0 ] + + IndexSnapshot + segments [ 0 1 ] + deleted [ 0-1 nil ] + + 3 - Index Batch [ C' ] + + segment 0 1 2 + numbers [ 1 2 3 ] [ 1 ] [ 1 ] + \_id [ A B C ] [ B ] [ C ] + + Compute bitset segment-0-deleted-by-2: + [ 0 0 1 ] + + OR it with previous ([ 0 1 0 ]) (call it 0-12) + [ 0 1 1 ] + + Compute bitset segment-1-deleted-by-2: + [ 0 ] + + OR it with previous (nil) + still just nil + + + IndexSnapshot + segments [ 0 1 2 ] + deleted [ 0-12 nil nil ] + ``` + +**is there opportunity to stop early when doc is found in one segment** +**also, more efficient way to find bits for long lists of ids?** + +### Searching + +In the bleve.index API all searching starts by getting an IndexReader, which represents a snapshot of the index at a point in time. + +As described in the section above, our index implementation maintains a pointer to the current IndexSnapshot. When a caller gets an IndexReader, they get a copy of this pointer, and can use it as long as they like. The IndexSnapshot contains SegmentSnapshots, which only contain pointers to immutable segments. The deleted posting lists associated with a segment change over time, but the particular deleted posting list in YOUR snapshot is immutable. This gives a stable view of the data. + +#### Term Search + +Term search is the only searching primitive exposed in today's bleve.index API. This ultimately could limit our ability to take advantage of the indexing improvements, but it also means it will be easier to get a first version of this working. + +A term search for term T in field F will look something like this: + +``` + searchResultPostings = empty + foreach segment { + dict := segment.Dictionary(F) + segmentResultPostings = dict.PostingsList(T, segmentSnapshotDeleted) + // make segmentLocal numbers into global numbers, and flip bits in searchResultPostings + } +``` + +The searchResultPostings will be a new implementation of the TermFieldReader inteface. + +As a reminder this interface is: + +``` +// TermFieldReader is the interface exposing the enumeration of documents +// containing a given term in a given field. Documents are returned in byte +// lexicographic order over their identifiers. +type TermFieldReader interface { + // Next returns the next document containing the term in this field, or nil + // when it reaches the end of the enumeration. The preAlloced TermFieldDoc + // is optional, and when non-nil, will be used instead of allocating memory. + Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error) + + // Advance resets the enumeration at specified document or its immediate + // follower. + Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error) + + // Count returns the number of documents contains the term in this field. + Count() uint64 + Close() error +} +``` + +At first glance this appears problematic, we have no way to return documents in order of their identifiers. But it turns out the wording of this perhaps too strong, or a bit ambiguous. Originally, this referred to the external identifiers, but with the introduction of a distinction between internal/external identifiers, returning them in order of their internal identifiers is also acceptable. **ASIDE**: the reason for this is that most callers just use Next() and literally don't care what the order is, they could be in any order and it would be fine. There is only one search that cares and that is the ConjunctionSearcher, which relies on Next/Advance having very specific semantics. Later in this document we will have a proposal to split into multiple interfaces: + +- The weakest interface, only supports Next() no ordering at all. +- Ordered, supporting Advance() +- And/Or'able capable of internally efficiently doing these ops with like interfaces (if not capable then can always fall back to external walking) + +But, the good news is that we don't even have to do that for our first implementation. As long as the global numbers we use for internal identifiers are consistent within this IndexSnapshot, then Next() will be ordered by ascending document number, and Advance() will still work correctly. + +NOTE: there is another place where we rely on the ordering of these hits, and that is in the "\_id" sort order. Previously this was the natural order, and a NOOP for the collector, now it must be implemented by actually sorting on the "\_id" field. We probably should introduce at least a marker interface to detect this. + +An ASCII art example: + +``` +Let's start with the IndexSnapshot we ended with earlier: + +3 - Index Batch [ C' ] + + segment 0 1 2 + numbers [ 1 2 3 ] [ 1 ] [ 1 ] + \_id [ A B C ] [ B ] [ C ] + + Compute bitset segment-0-deleted-by-2: + [ 0 0 1 ] + + OR it with previous ([ 0 1 0 ]) (call it 0-12) + [ 0 1 1 ] + +Compute bitset segment-1-deleted-by-2: + [ 0 0 0 ] + +OR it with previous (nil) + still just nil + + + IndexSnapshot + segments [ 0 1 2 ] + deleted [ 0-12 nil nil ] + +Now let's search for the term 'cat' in the field 'desc' and let's assume that Document C (both versions) would match it. + +Concurrently: + + - Segment 0 + - Get Term Dictionary For Field 'desc' + - From it get Postings List for term 'cat' EXCLUDING 0-12 + - raw segment matches [ 0 0 1 ] but excluding [ 0 1 1 ] gives [ 0 0 0 ] + - Segment 1 + - Get Term Dictionary For Field 'desc' + - From it get Postings List for term 'cat' excluding nil + - [ 0 ] + - Segment 2 + - Get Term Dictionary For Field 'desc' + - From it get Postings List for term 'cat' excluding nil + - [ 1 ] + +Map local bitsets into global number space (global meaning cross-segment but still unique to this snapshot) + +IndexSnapshot already should have mapping something like: +0 - Offset 0 +1 - Offset 3 (because segment 0 had 3 docs) +2 - Offset 4 (becuase segment 1 had 1 doc) + +This maps to search result bitset: + +[ 0 0 0 0 1] + +Caller would call Next() and get doc number 5 (assuming 1 based indexing for now) + +Caller could then ask to get term locations, stored fields, external doc ID for document number 5. Internally in the IndexSnapshot, we can now convert that back, and realize doc number 5 comes from segment 2, 5-4=1 so we're looking for doc number 1 in segment 2. That happens to be C... + +``` + +#### Future improvements + +In the future, interfaces to detect these non-serially operating TermFieldReaders could expose their own And() and Or() up to the higher level Conjunction/Disjunction searchers. Doing this alone offers some win, but also means there would be greater burden on the Searcher code rewriting logical expressions for maximum performance. + +Another related topic is that of peak memory usage. With serially operating TermFieldReaders it was necessary to start them all at the same time and operate in unison. However, with these non-serially operating TermFieldReaders we have the option of doing a few at a time, consolidating them, dispoting the intermediaries, and then doing a few more. For very complex queries with many clauses this could reduce peak memory usage. + + +### Memory Tracking + +All segments must be able to produce two statistics, an estimate of their explicit memory usage, and their actual size on disk (if any). For in-memory segments, disk usage could be zero, and the memory usage represents the entire information content. For mmap-based disk segments, the memory could be as low as the size of tracking structure itself (say just a few pointers). + +This would allow the implementation to throttle or block incoming mutations when a threshold memory usage has (or would be) exceeded. + +### Persistence + +Obviously, we want to support (but maybe not require) asynchronous persistence of segments. My expectation is that segments are initially built in memory. At some point they are persisted to disk. This poses some interesting challenges. + +At runtime, the state of an index (it's IndexSnapshot) is not only the contents of the segments, but also the bitmasks of deleted documents. These bitmasks indirectly encode an ordering in which the segments were added. The reason is that the bitmasks encode which items have been obsoleted by other (subsequent or more future) segments. In the runtime implementation we compute bitmask deltas and then merge them at the same time we bring the new segment in. One idea is that we could take a similar approach on disk. When we persist a segment, we persist the bitmask deltas of segments known to exist at that time, and eventually these can get merged up into a base segment deleted bitmask. + +This also relates to the topic rollback, addressed next... + + +### Rollback + +One desirable property in the Couchbase ecosystem is the ability to rollback to some previous (though typically not long ago) state. One idea for keeping this property in this design is to protect some of the most recent segments from merging. Then, if necessary, they could be "undone" to reveal previous states of the system. In these scenarios "undone" has to properly undo the deleted bitmasks on the other segments. Again, the current thinking is that rather than "undo" anything, it could be work that was deferred in the first place, thus making it easier to logically undo. + +Another possibly related approach would be to tie this into our existing snapshot mechanism. Perhaps simulating a slow reader (holding onto index snapshots) for some period of time, can be the mechanism to achieve the desired end goal. + + +### Internal Storage + +The bleve.index API has support for "internal storage". The ability to store information under a separate name space. + +This is not used for high volume storage, so it is tempting to think we could just put a small k/v store alongside the rest of the index. But, the reality is that this storage is used to maintain key information related to the rollback scenario. Because of this, its crucial that ordering and overwriting of key/value pairs correspond with actual segment persistence in the index. Based on this, I believe its important to put the internal key/value pairs inside the segments themselves. But, this also means that they must follow a similar "deleted" bitmask approach to obsolete values in older segments. But, this also seems to substantially increase the complexity of the solution because of the separate name space, it would appear to require its own bitmask. Further keys aren't numeric, which then implies yet another mapping from internal key to number, etc. + +More thought is required here. + +### Merging + +The segmented index approach requires merging to prevent the number of segments from growing too large. + +Recent experience with LSMs has taught us that having the correct merge strategy can make a huge difference in the overall performance of the system. In particular, a simple merge strategy which merges segments too aggressively can lead to high write amplification and unnecessarily rendering cached data useless. + +A few simple principles have been identified. + +- Roughly we merge multiple smaller segments into a single larger one. +- The larger a segment gets the less likely we should be to ever merge it. +- Segments with large numbers of deleted/obsoleted items are good candidates as the merge will result in a space savings. +- Segments with all items deleted/obsoleted can be dropped. + +Merging of a segment should be able to proceed even if that segment is held by an ongoing snapshot, it should only delay the removal of it. diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/event.go b/vendor/github.com/blevesearch/bleve/index/scorch/event.go new file mode 100644 index 00000000..dd79d6d0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/event.go @@ -0,0 +1,56 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import "time" + +// RegistryAsyncErrorCallbacks should be treated as read-only after +// process init()'ialization. +var RegistryAsyncErrorCallbacks = map[string]func(error){} + +// RegistryEventCallbacks should be treated as read-only after +// process init()'ialization. +var RegistryEventCallbacks = map[string]func(Event){} + +// Event represents the information provided in an OnEvent() callback. +type Event struct { + Kind EventKind + Scorch *Scorch + Duration time.Duration +} + +// EventKind represents an event code for OnEvent() callbacks. +type EventKind int + +// EventKindCloseStart is fired when a Scorch.Close() has begun. +var EventKindCloseStart = EventKind(1) + +// EventKindClose is fired when a scorch index has been fully closed. +var EventKindClose = EventKind(2) + +// EventKindMergerProgress is fired when the merger has completed a +// round of merge processing. +var EventKindMergerProgress = EventKind(3) + +// EventKindPersisterProgress is fired when the persister has completed +// a round of persistence processing. +var EventKindPersisterProgress = EventKind(4) + +// EventKindBatchIntroductionStart is fired when Batch() is invoked which +// introduces a new segment. +var EventKindBatchIntroductionStart = EventKind(5) + +// EventKindBatchIntroduction is fired when Batch() completes. +var EventKindBatchIntroduction = EventKind(6) diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go b/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go new file mode 100644 index 00000000..fb6afd5d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go @@ -0,0 +1,443 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "sync/atomic" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index/scorch/segment" +) + +type segmentIntroduction struct { + id uint64 + data segment.Segment + obsoletes map[uint64]*roaring.Bitmap + ids []string + internal map[string][]byte + + applied chan error + persisted chan error +} + +type persistIntroduction struct { + persisted map[uint64]segment.Segment + applied notificationChan +} + +type epochWatcher struct { + epoch uint64 + notifyCh notificationChan +} + +type snapshotReversion struct { + snapshot *IndexSnapshot + applied chan error + persisted chan error +} + +func (s *Scorch) mainLoop() { + var epochWatchers []*epochWatcher +OUTER: + for { + atomic.AddUint64(&s.stats.TotIntroduceLoop, 1) + + select { + case <-s.closeCh: + break OUTER + + case epochWatcher := <-s.introducerNotifier: + epochWatchers = append(epochWatchers, epochWatcher) + + case nextMerge := <-s.merges: + s.introduceMerge(nextMerge) + + case next := <-s.introductions: + err := s.introduceSegment(next) + if err != nil { + continue OUTER + } + + case persist := <-s.persists: + s.introducePersist(persist) + + case revertTo := <-s.revertToSnapshots: + err := s.revertToSnapshot(revertTo) + if err != nil { + continue OUTER + } + } + + var epochCurr uint64 + s.rootLock.RLock() + if s.root != nil { + epochCurr = s.root.epoch + } + s.rootLock.RUnlock() + var epochWatchersNext []*epochWatcher + for _, w := range epochWatchers { + if w.epoch < epochCurr { + close(w.notifyCh) + } else { + epochWatchersNext = append(epochWatchersNext, w) + } + } + epochWatchers = epochWatchersNext + } + + s.asyncTasks.Done() +} + +func (s *Scorch) introduceSegment(next *segmentIntroduction) error { + atomic.AddUint64(&s.stats.TotIntroduceSegmentBeg, 1) + defer atomic.AddUint64(&s.stats.TotIntroduceSegmentEnd, 1) + + s.rootLock.RLock() + root := s.root + s.rootLock.RUnlock() + + nsegs := len(root.segment) + + // prepare new index snapshot + newSnapshot := &IndexSnapshot{ + parent: s, + segment: make([]*SegmentSnapshot, 0, nsegs+1), + offsets: make([]uint64, 0, nsegs+1), + internal: make(map[string][]byte, len(root.internal)), + refs: 1, + creator: "introduceSegment", + } + + // iterate through current segments + var running uint64 + for i := range root.segment { + // see if optimistic work included this segment + delta, ok := next.obsoletes[root.segment[i].id] + if !ok { + var err error + delta, err = root.segment[i].segment.DocNumbers(next.ids) + if err != nil { + next.applied <- fmt.Errorf("error computing doc numbers: %v", err) + close(next.applied) + _ = newSnapshot.DecRef() + return err + } + } + + newss := &SegmentSnapshot{ + id: root.segment[i].id, + segment: root.segment[i].segment, + cachedDocs: root.segment[i].cachedDocs, + creator: root.segment[i].creator, + } + + // apply new obsoletions + if root.segment[i].deleted == nil { + newss.deleted = delta + } else { + newss.deleted = roaring.Or(root.segment[i].deleted, delta) + } + if newss.deleted.IsEmpty() { + newss.deleted = nil + } + + // check for live size before copying + if newss.LiveSize() > 0 { + newSnapshot.segment = append(newSnapshot.segment, newss) + root.segment[i].segment.AddRef() + newSnapshot.offsets = append(newSnapshot.offsets, running) + running += newss.segment.Count() + } + } + + // append new segment, if any, to end of the new index snapshot + if next.data != nil { + newSegmentSnapshot := &SegmentSnapshot{ + id: next.id, + segment: next.data, // take ownership of next.data's ref-count + cachedDocs: &cachedDocs{cache: nil}, + creator: "introduceSegment", + } + newSnapshot.segment = append(newSnapshot.segment, newSegmentSnapshot) + newSnapshot.offsets = append(newSnapshot.offsets, running) + + // increment numItemsIntroduced which tracks the number of items + // queued for persistence. + atomic.AddUint64(&s.stats.TotIntroducedItems, newSegmentSnapshot.Count()) + atomic.AddUint64(&s.stats.TotIntroducedSegmentsBatch, 1) + } + // copy old values + for key, oldVal := range root.internal { + newSnapshot.internal[key] = oldVal + } + // set new values and apply deletes + for key, newVal := range next.internal { + if newVal != nil { + newSnapshot.internal[key] = newVal + } else { + delete(newSnapshot.internal, key) + } + } + + newSnapshot.updateSize() + s.rootLock.Lock() + if next.persisted != nil { + s.rootPersisted = append(s.rootPersisted, next.persisted) + } + // swap in new index snapshot + newSnapshot.epoch = s.nextSnapshotEpoch + s.nextSnapshotEpoch++ + rootPrev := s.root + s.root = newSnapshot + // release lock + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + close(next.applied) + + return nil +} + +func (s *Scorch) introducePersist(persist *persistIntroduction) { + atomic.AddUint64(&s.stats.TotIntroducePersistBeg, 1) + defer atomic.AddUint64(&s.stats.TotIntroducePersistEnd, 1) + + s.rootLock.Lock() + root := s.root + nextSnapshotEpoch := s.nextSnapshotEpoch + s.nextSnapshotEpoch++ + s.rootLock.Unlock() + + newIndexSnapshot := &IndexSnapshot{ + parent: s, + epoch: nextSnapshotEpoch, + segment: make([]*SegmentSnapshot, len(root.segment)), + offsets: make([]uint64, len(root.offsets)), + internal: make(map[string][]byte, len(root.internal)), + refs: 1, + creator: "introducePersist", + } + + for i, segmentSnapshot := range root.segment { + // see if this segment has been replaced + if replacement, ok := persist.persisted[segmentSnapshot.id]; ok { + newSegmentSnapshot := &SegmentSnapshot{ + id: segmentSnapshot.id, + segment: replacement, + deleted: segmentSnapshot.deleted, + cachedDocs: segmentSnapshot.cachedDocs, + creator: "introducePersist", + } + newIndexSnapshot.segment[i] = newSegmentSnapshot + delete(persist.persisted, segmentSnapshot.id) + + // update items persisted incase of a new segment snapshot + atomic.AddUint64(&s.stats.TotPersistedItems, newSegmentSnapshot.Count()) + atomic.AddUint64(&s.stats.TotPersistedSegments, 1) + } else { + newIndexSnapshot.segment[i] = root.segment[i] + newIndexSnapshot.segment[i].segment.AddRef() + } + newIndexSnapshot.offsets[i] = root.offsets[i] + } + + for k, v := range root.internal { + newIndexSnapshot.internal[k] = v + } + + newIndexSnapshot.updateSize() + s.rootLock.Lock() + rootPrev := s.root + s.root = newIndexSnapshot + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + close(persist.applied) +} + +func (s *Scorch) introduceMerge(nextMerge *segmentMerge) { + atomic.AddUint64(&s.stats.TotIntroduceMergeBeg, 1) + defer atomic.AddUint64(&s.stats.TotIntroduceMergeEnd, 1) + + s.rootLock.RLock() + root := s.root + s.rootLock.RUnlock() + + newSnapshot := &IndexSnapshot{ + parent: s, + internal: root.internal, + refs: 1, + creator: "introduceMerge", + } + + // iterate through current segments + newSegmentDeleted := roaring.NewBitmap() + var running uint64 + for i := range root.segment { + segmentID := root.segment[i].id + if segSnapAtMerge, ok := nextMerge.old[segmentID]; ok { + // this segment is going away, see if anything else was deleted since we started the merge + if segSnapAtMerge != nil && root.segment[i].deleted != nil { + // assume all these deletes are new + deletedSince := root.segment[i].deleted + // if we already knew about some of them, remove + if segSnapAtMerge.deleted != nil { + deletedSince = roaring.AndNot(root.segment[i].deleted, segSnapAtMerge.deleted) + } + deletedSinceItr := deletedSince.Iterator() + for deletedSinceItr.HasNext() { + oldDocNum := deletedSinceItr.Next() + newDocNum := nextMerge.oldNewDocNums[segmentID][oldDocNum] + newSegmentDeleted.Add(uint32(newDocNum)) + } + } + // clean up the old segment map to figure out the + // obsolete segments wrt root in meantime, whatever + // segments left behind in old map after processing + // the root segments would be the obsolete segment set + delete(nextMerge.old, segmentID) + } else if root.segment[i].LiveSize() > 0 { + // this segment is staying + newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{ + id: root.segment[i].id, + segment: root.segment[i].segment, + deleted: root.segment[i].deleted, + cachedDocs: root.segment[i].cachedDocs, + creator: root.segment[i].creator, + }) + root.segment[i].segment.AddRef() + newSnapshot.offsets = append(newSnapshot.offsets, running) + running += root.segment[i].segment.Count() + } + } + + // before the newMerge introduction, need to clean the newly + // merged segment wrt the current root segments, hence + // applying the obsolete segment contents to newly merged segment + for segID, ss := range nextMerge.old { + obsoleted := ss.DocNumbersLive() + if obsoleted != nil { + obsoletedIter := obsoleted.Iterator() + for obsoletedIter.HasNext() { + oldDocNum := obsoletedIter.Next() + newDocNum := nextMerge.oldNewDocNums[segID][oldDocNum] + newSegmentDeleted.Add(uint32(newDocNum)) + } + } + } + // In case where all the docs in the newly merged segment getting + // deleted by the time we reach here, can skip the introduction. + if nextMerge.new != nil && + nextMerge.new.Count() > newSegmentDeleted.GetCardinality() { + // put new segment at end + newSnapshot.segment = append(newSnapshot.segment, &SegmentSnapshot{ + id: nextMerge.id, + segment: nextMerge.new, // take ownership for nextMerge.new's ref-count + deleted: newSegmentDeleted, + cachedDocs: &cachedDocs{cache: nil}, + creator: "introduceMerge", + }) + newSnapshot.offsets = append(newSnapshot.offsets, running) + atomic.AddUint64(&s.stats.TotIntroducedSegmentsMerge, 1) + } + + newSnapshot.AddRef() // 1 ref for the nextMerge.notify response + + newSnapshot.updateSize() + s.rootLock.Lock() + // swap in new index snapshot + newSnapshot.epoch = s.nextSnapshotEpoch + s.nextSnapshotEpoch++ + rootPrev := s.root + s.root = newSnapshot + // release lock + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + // notify requester that we incorporated this + nextMerge.notify <- newSnapshot + close(nextMerge.notify) +} + +func (s *Scorch) revertToSnapshot(revertTo *snapshotReversion) error { + atomic.AddUint64(&s.stats.TotIntroduceRevertBeg, 1) + defer atomic.AddUint64(&s.stats.TotIntroduceRevertEnd, 1) + + if revertTo.snapshot == nil { + err := fmt.Errorf("Cannot revert to a nil snapshot") + revertTo.applied <- err + return err + } + + // acquire lock + s.rootLock.Lock() + + // prepare a new index snapshot, based on next snapshot + newSnapshot := &IndexSnapshot{ + parent: s, + segment: make([]*SegmentSnapshot, len(revertTo.snapshot.segment)), + offsets: revertTo.snapshot.offsets, + internal: revertTo.snapshot.internal, + epoch: s.nextSnapshotEpoch, + refs: 1, + creator: "revertToSnapshot", + } + s.nextSnapshotEpoch++ + + // iterate through segments + for i, segmentSnapshot := range revertTo.snapshot.segment { + newSnapshot.segment[i] = &SegmentSnapshot{ + id: segmentSnapshot.id, + segment: segmentSnapshot.segment, + deleted: segmentSnapshot.deleted, + cachedDocs: segmentSnapshot.cachedDocs, + creator: segmentSnapshot.creator, + } + newSnapshot.segment[i].segment.AddRef() + + // remove segment from ineligibleForRemoval map + filename := zapFileName(segmentSnapshot.id) + delete(s.ineligibleForRemoval, filename) + } + + if revertTo.persisted != nil { + s.rootPersisted = append(s.rootPersisted, revertTo.persisted) + } + + newSnapshot.updateSize() + // swap in new snapshot + rootPrev := s.root + s.root = newSnapshot + // release lock + s.rootLock.Unlock() + + if rootPrev != nil { + _ = rootPrev.DecRef() + } + + close(revertTo.applied) + + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/merge.go b/vendor/github.com/blevesearch/bleve/index/scorch/merge.go new file mode 100644 index 00000000..41b734aa --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/merge.go @@ -0,0 +1,342 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "encoding/json" + "fmt" + "os" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index/scorch/mergeplan" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/index/scorch/segment/zap" +) + +func (s *Scorch) mergerLoop() { + var lastEpochMergePlanned uint64 + mergePlannerOptions, err := s.parseMergePlannerOptions() + if err != nil { + s.fireAsyncError(fmt.Errorf("mergePlannerOption json parsing err: %v", err)) + s.asyncTasks.Done() + return + } + +OUTER: + for { + atomic.AddUint64(&s.stats.TotFileMergeLoopBeg, 1) + + select { + case <-s.closeCh: + break OUTER + + default: + // check to see if there is a new snapshot to persist + s.rootLock.RLock() + ourSnapshot := s.root + ourSnapshot.AddRef() + atomic.StoreUint64(&s.iStats.mergeSnapshotSize, uint64(ourSnapshot.Size())) + atomic.StoreUint64(&s.iStats.mergeEpoch, ourSnapshot.epoch) + s.rootLock.RUnlock() + + if ourSnapshot.epoch != lastEpochMergePlanned { + startTime := time.Now() + + // lets get started + err := s.planMergeAtSnapshot(ourSnapshot, mergePlannerOptions) + if err != nil { + atomic.StoreUint64(&s.iStats.mergeEpoch, 0) + if err == ErrClosed { + // index has been closed + _ = ourSnapshot.DecRef() + break OUTER + } + s.fireAsyncError(fmt.Errorf("merging err: %v", err)) + _ = ourSnapshot.DecRef() + atomic.AddUint64(&s.stats.TotFileMergeLoopErr, 1) + continue OUTER + } + lastEpochMergePlanned = ourSnapshot.epoch + + s.fireEvent(EventKindMergerProgress, time.Since(startTime)) + } + _ = ourSnapshot.DecRef() + + // tell the persister we're waiting for changes + // first make a epochWatcher chan + ew := &epochWatcher{ + epoch: lastEpochMergePlanned, + notifyCh: make(notificationChan, 1), + } + + // give it to the persister + select { + case <-s.closeCh: + break OUTER + case s.persisterNotifier <- ew: + } + + // now wait for persister (but also detect close) + select { + case <-s.closeCh: + break OUTER + case <-ew.notifyCh: + } + } + + atomic.AddUint64(&s.stats.TotFileMergeLoopEnd, 1) + } + + s.asyncTasks.Done() +} + +func (s *Scorch) parseMergePlannerOptions() (*mergeplan.MergePlanOptions, + error) { + mergePlannerOptions := mergeplan.DefaultMergePlanOptions + if v, ok := s.config["scorchMergePlanOptions"]; ok { + b, err := json.Marshal(v) + if err != nil { + return &mergePlannerOptions, err + } + + err = json.Unmarshal(b, &mergePlannerOptions) + if err != nil { + return &mergePlannerOptions, err + } + + err = mergeplan.ValidateMergePlannerOptions(&mergePlannerOptions) + if err != nil { + return nil, err + } + } + return &mergePlannerOptions, nil +} + +func (s *Scorch) planMergeAtSnapshot(ourSnapshot *IndexSnapshot, + options *mergeplan.MergePlanOptions) error { + // build list of zap segments in this snapshot + var onlyZapSnapshots []mergeplan.Segment + for _, segmentSnapshot := range ourSnapshot.segment { + if _, ok := segmentSnapshot.segment.(*zap.Segment); ok { + onlyZapSnapshots = append(onlyZapSnapshots, segmentSnapshot) + } + } + + atomic.AddUint64(&s.stats.TotFileMergePlan, 1) + + // give this list to the planner + resultMergePlan, err := mergeplan.Plan(onlyZapSnapshots, options) + if err != nil { + atomic.AddUint64(&s.stats.TotFileMergePlanErr, 1) + return fmt.Errorf("merge planning err: %v", err) + } + if resultMergePlan == nil { + // nothing to do + atomic.AddUint64(&s.stats.TotFileMergePlanNone, 1) + return nil + } + + atomic.AddUint64(&s.stats.TotFileMergePlanOk, 1) + + atomic.AddUint64(&s.stats.TotFileMergePlanTasks, uint64(len(resultMergePlan.Tasks))) + + // process tasks in serial for now + var notifications []chan *IndexSnapshot + for _, task := range resultMergePlan.Tasks { + if len(task.Segments) == 0 { + atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegmentsEmpty, 1) + continue + } + + atomic.AddUint64(&s.stats.TotFileMergePlanTasksSegments, uint64(len(task.Segments))) + + oldMap := make(map[uint64]*SegmentSnapshot) + newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1) + segmentsToMerge := make([]*zap.Segment, 0, len(task.Segments)) + docsToDrop := make([]*roaring.Bitmap, 0, len(task.Segments)) + + for _, planSegment := range task.Segments { + if segSnapshot, ok := planSegment.(*SegmentSnapshot); ok { + oldMap[segSnapshot.id] = segSnapshot + if zapSeg, ok := segSnapshot.segment.(*zap.Segment); ok { + if segSnapshot.LiveSize() == 0 { + atomic.AddUint64(&s.stats.TotFileMergeSegmentsEmpty, 1) + oldMap[segSnapshot.id] = nil + } else { + segmentsToMerge = append(segmentsToMerge, zapSeg) + docsToDrop = append(docsToDrop, segSnapshot.deleted) + } + } + } + } + + var oldNewDocNums map[uint64][]uint64 + var segment segment.Segment + if len(segmentsToMerge) > 0 { + filename := zapFileName(newSegmentID) + s.markIneligibleForRemoval(filename) + path := s.path + string(os.PathSeparator) + filename + + fileMergeZapStartTime := time.Now() + + atomic.AddUint64(&s.stats.TotFileMergeZapBeg, 1) + newDocNums, nBytes, err := zap.Merge(segmentsToMerge, docsToDrop, path, DefaultChunkFactor) + atomic.AddUint64(&s.stats.TotFileMergeZapEnd, 1) + atomic.AddUint64(&s.stats.TotFileMergeWrittenBytes, nBytes) + + fileMergeZapTime := uint64(time.Since(fileMergeZapStartTime)) + atomic.AddUint64(&s.stats.TotFileMergeZapTime, fileMergeZapTime) + if atomic.LoadUint64(&s.stats.MaxFileMergeZapTime) < fileMergeZapTime { + atomic.StoreUint64(&s.stats.MaxFileMergeZapTime, fileMergeZapTime) + } + + if err != nil { + s.unmarkIneligibleForRemoval(filename) + atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) + return fmt.Errorf("merging failed: %v", err) + } + + segment, err = zap.Open(path) + if err != nil { + s.unmarkIneligibleForRemoval(filename) + atomic.AddUint64(&s.stats.TotFileMergePlanTasksErr, 1) + return err + } + oldNewDocNums = make(map[uint64][]uint64) + for i, segNewDocNums := range newDocNums { + oldNewDocNums[task.Segments[i].Id()] = segNewDocNums + } + + atomic.AddUint64(&s.stats.TotFileMergeSegments, uint64(len(segmentsToMerge))) + } + + sm := &segmentMerge{ + id: newSegmentID, + old: oldMap, + oldNewDocNums: oldNewDocNums, + new: segment, + notify: make(chan *IndexSnapshot, 1), + } + notifications = append(notifications, sm.notify) + + // give it to the introducer + select { + case <-s.closeCh: + _ = segment.Close() + return ErrClosed + case s.merges <- sm: + atomic.AddUint64(&s.stats.TotFileMergeIntroductions, 1) + } + + atomic.AddUint64(&s.stats.TotFileMergePlanTasksDone, 1) + } + + for _, notification := range notifications { + select { + case <-s.closeCh: + return ErrClosed + case newSnapshot := <-notification: + atomic.AddUint64(&s.stats.TotFileMergeIntroductionsDone, 1) + if newSnapshot != nil { + _ = newSnapshot.DecRef() + } + } + } + + return nil +} + +type segmentMerge struct { + id uint64 + old map[uint64]*SegmentSnapshot + oldNewDocNums map[uint64][]uint64 + new segment.Segment + notify chan *IndexSnapshot +} + +// perform a merging of the given SegmentBase instances into a new, +// persisted segment, and synchronously introduce that new segment +// into the root +func (s *Scorch) mergeSegmentBases(snapshot *IndexSnapshot, + sbs []*zap.SegmentBase, sbsDrops []*roaring.Bitmap, sbsIndexes []int, + chunkFactor uint32) (*IndexSnapshot, uint64, error) { + atomic.AddUint64(&s.stats.TotMemMergeBeg, 1) + + memMergeZapStartTime := time.Now() + + atomic.AddUint64(&s.stats.TotMemMergeZapBeg, 1) + + newSegmentID := atomic.AddUint64(&s.nextSegmentID, 1) + filename := zapFileName(newSegmentID) + path := s.path + string(os.PathSeparator) + filename + + newDocNums, _, err := + zap.MergeSegmentBases(sbs, sbsDrops, path, chunkFactor) + + atomic.AddUint64(&s.stats.TotMemMergeZapEnd, 1) + + memMergeZapTime := uint64(time.Since(memMergeZapStartTime)) + atomic.AddUint64(&s.stats.TotMemMergeZapTime, memMergeZapTime) + if atomic.LoadUint64(&s.stats.MaxMemMergeZapTime) < memMergeZapTime { + atomic.StoreUint64(&s.stats.MaxMemMergeZapTime, memMergeZapTime) + } + + if err != nil { + atomic.AddUint64(&s.stats.TotMemMergeErr, 1) + return nil, 0, err + } + + segment, err := zap.Open(path) + if err != nil { + atomic.AddUint64(&s.stats.TotMemMergeErr, 1) + return nil, 0, err + } + + // update persisted stats + atomic.AddUint64(&s.stats.TotPersistedItems, segment.Count()) + atomic.AddUint64(&s.stats.TotPersistedSegments, 1) + + sm := &segmentMerge{ + id: newSegmentID, + old: make(map[uint64]*SegmentSnapshot), + oldNewDocNums: make(map[uint64][]uint64), + new: segment, + notify: make(chan *IndexSnapshot, 1), + } + + for i, idx := range sbsIndexes { + ss := snapshot.segment[idx] + sm.old[ss.id] = ss + sm.oldNewDocNums[ss.id] = newDocNums[i] + } + + select { // send to introducer + case <-s.closeCh: + _ = segment.DecRef() + return nil, 0, ErrClosed + case s.merges <- sm: + } + + select { // wait for introduction to complete + case <-s.closeCh: + return nil, 0, ErrClosed + case newSnapshot := <-sm.notify: + atomic.AddUint64(&s.stats.TotMemMergeSegments, uint64(len(sbs))) + atomic.AddUint64(&s.stats.TotMemMergeDone, 1) + return newSnapshot, newSegmentID, nil + } +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/merge_plan.go b/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/merge_plan.go new file mode 100644 index 00000000..b09e5381 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/merge_plan.go @@ -0,0 +1,386 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mergeplan provides a segment merge planning approach that's +// inspired by Lucene's TieredMergePolicy.java and descriptions like +// http://blog.mikemccandless.com/2011/02/visualizing-lucenes-segment-merges.html +package mergeplan + +import ( + "errors" + "fmt" + "math" + "sort" + "strings" +) + +// A Segment represents the information that the planner needs to +// calculate segment merging. +type Segment interface { + // Unique id of the segment -- used for sorting. + Id() uint64 + + // Full segment size (the size before any logical deletions). + FullSize() int64 + + // Size of the live data of the segment; i.e., FullSize() minus + // any logical deletions. + LiveSize() int64 +} + +// Plan() will functionally compute a merge plan. A segment will be +// assigned to at most a single MergeTask in the output MergePlan. A +// segment not assigned to any MergeTask means the segment should +// remain unmerged. +func Plan(segments []Segment, o *MergePlanOptions) (*MergePlan, error) { + return plan(segments, o) +} + +// A MergePlan is the result of the Plan() API. +// +// The planner doesn’t know how or whether these tasks are executed -- +// that’s up to a separate merge execution system, which might execute +// these tasks concurrently or not, and which might execute all the +// tasks or not. +type MergePlan struct { + Tasks []*MergeTask +} + +// A MergeTask represents several segments that should be merged +// together into a single segment. +type MergeTask struct { + Segments []Segment +} + +// The MergePlanOptions is designed to be reusable between planning calls. +type MergePlanOptions struct { + // Max # segments per logarithmic tier, or max width of any + // logarithmic “step”. Smaller values mean more merging but fewer + // segments. Should be >= SegmentsPerMergeTask, else you'll have + // too much merging. + MaxSegmentsPerTier int + + // Max size of any segment produced after merging. Actual + // merging, however, may produce segment sizes different than the + // planner’s predicted sizes. + MaxSegmentSize int64 + + // The growth factor for each tier in a staircase of idealized + // segments computed by CalcBudget(). + TierGrowth float64 + + // The number of segments in any resulting MergeTask. e.g., + // len(result.Tasks[ * ].Segments) == SegmentsPerMergeTask. + SegmentsPerMergeTask int + + // Small segments are rounded up to this size, i.e., treated as + // equal (floor) size for consideration. This is to prevent lots + // of tiny segments from resulting in a long tail in the index. + FloorSegmentSize int64 + + // Controls how aggressively merges that reclaim more deletions + // are favored. Higher values will more aggressively target + // merges that reclaim deletions, but be careful not to go so high + // that way too much merging takes place; a value of 3.0 is + // probably nearly too high. A value of 0.0 means deletions don't + // impact merge selection. + ReclaimDeletesWeight float64 + + // Optional, defaults to mergeplan.CalcBudget(). + CalcBudget func(totalSize int64, firstTierSize int64, + o *MergePlanOptions) (budgetNumSegments int) + + // Optional, defaults to mergeplan.ScoreSegments(). + ScoreSegments func(segments []Segment, o *MergePlanOptions) float64 + + // Optional. + Logger func(string) +} + +// Returns the higher of the input or FloorSegmentSize. +func (o *MergePlanOptions) RaiseToFloorSegmentSize(s int64) int64 { + if s > o.FloorSegmentSize { + return s + } + return o.FloorSegmentSize +} + +// MaxSegmentSizeLimit represents the maximum size of a segment, +// this limit comes with hit-1 optimisation/max encoding limit uint31. +const MaxSegmentSizeLimit = 1<<31 - 1 + +// ErrMaxSegmentSizeTooLarge is returned when the size of the segment +// exceeds the MaxSegmentSizeLimit +var ErrMaxSegmentSizeTooLarge = errors.New("MaxSegmentSize exceeds the size limit") + +// DefaultMergePlanOptions suggests the default options. +var DefaultMergePlanOptions = MergePlanOptions{ + MaxSegmentsPerTier: 10, + MaxSegmentSize: 5000000, + TierGrowth: 10.0, + SegmentsPerMergeTask: 10, + FloorSegmentSize: 2000, + ReclaimDeletesWeight: 2.0, +} + +// ------------------------------------------- + +func plan(segmentsIn []Segment, o *MergePlanOptions) (*MergePlan, error) { + if len(segmentsIn) <= 1 { + return nil, nil + } + + if o == nil { + o = &DefaultMergePlanOptions + } + + segments := append([]Segment(nil), segmentsIn...) // Copy. + + sort.Sort(byLiveSizeDescending(segments)) + + var minLiveSize int64 = math.MaxInt64 + + var eligibles []Segment + var eligiblesLiveSize int64 + + for _, segment := range segments { + if minLiveSize > segment.LiveSize() { + minLiveSize = segment.LiveSize() + } + + // Only small-enough segments are eligible. + if segment.LiveSize() < o.MaxSegmentSize/2 { + eligibles = append(eligibles, segment) + eligiblesLiveSize += segment.LiveSize() + } + } + + minLiveSize = o.RaiseToFloorSegmentSize(minLiveSize) + + calcBudget := o.CalcBudget + if calcBudget == nil { + calcBudget = CalcBudget + } + + budgetNumSegments := CalcBudget(eligiblesLiveSize, minLiveSize, o) + + scoreSegments := o.ScoreSegments + if scoreSegments == nil { + scoreSegments = ScoreSegments + } + + rv := &MergePlan{} + + var empties []Segment + for _, eligible := range eligibles { + if eligible.LiveSize() <= 0 { + empties = append(empties, eligible) + } + } + if len(empties) > 0 { + rv.Tasks = append(rv.Tasks, &MergeTask{Segments: empties}) + eligibles = removeSegments(eligibles, empties) + } + + // While we’re over budget, keep looping, which might produce + // another MergeTask. + for len(eligibles) > 0 && (len(eligibles)+len(rv.Tasks)) > budgetNumSegments { + // Track a current best roster as we examine and score + // potential rosters of merges. + var bestRoster []Segment + var bestRosterScore float64 // Lower score is better. + + for startIdx := 0; startIdx < len(eligibles); startIdx++ { + var roster []Segment + var rosterLiveSize int64 + + for idx := startIdx; idx < len(eligibles) && len(roster) < o.SegmentsPerMergeTask; idx++ { + eligible := eligibles[idx] + + if rosterLiveSize+eligible.LiveSize() < o.MaxSegmentSize { + roster = append(roster, eligible) + rosterLiveSize += eligible.LiveSize() + } + } + + if len(roster) > 0 { + rosterScore := scoreSegments(roster, o) + + if len(bestRoster) <= 0 || rosterScore < bestRosterScore { + bestRoster = roster + bestRosterScore = rosterScore + } + } + } + + if len(bestRoster) <= 0 { + return rv, nil + } + + rv.Tasks = append(rv.Tasks, &MergeTask{Segments: bestRoster}) + + eligibles = removeSegments(eligibles, bestRoster) + } + + return rv, nil +} + +// Compute the number of segments that would be needed to cover the +// totalSize, by climbing up a logarithmically growing staircase of +// segment tiers. +func CalcBudget(totalSize int64, firstTierSize int64, o *MergePlanOptions) ( + budgetNumSegments int) { + tierSize := firstTierSize + if tierSize < 1 { + tierSize = 1 + } + + maxSegmentsPerTier := o.MaxSegmentsPerTier + if maxSegmentsPerTier < 1 { + maxSegmentsPerTier = 1 + } + + tierGrowth := o.TierGrowth + if tierGrowth < 1.0 { + tierGrowth = 1.0 + } + + for totalSize > 0 { + segmentsInTier := float64(totalSize) / float64(tierSize) + if segmentsInTier < float64(maxSegmentsPerTier) { + budgetNumSegments += int(math.Ceil(segmentsInTier)) + break + } + + budgetNumSegments += maxSegmentsPerTier + totalSize -= int64(maxSegmentsPerTier) * tierSize + tierSize = int64(float64(tierSize) * tierGrowth) + } + + return budgetNumSegments +} + +// Of note, removeSegments() keeps the ordering of the results stable. +func removeSegments(segments []Segment, toRemove []Segment) []Segment { + rv := make([]Segment, 0, len(segments)-len(toRemove)) +OUTER: + for _, segment := range segments { + for _, r := range toRemove { + if segment == r { + continue OUTER + } + } + rv = append(rv, segment) + } + return rv +} + +// Smaller result score is better. +func ScoreSegments(segments []Segment, o *MergePlanOptions) float64 { + var totBeforeSize int64 + var totAfterSize int64 + var totAfterSizeFloored int64 + + for _, segment := range segments { + totBeforeSize += segment.FullSize() + totAfterSize += segment.LiveSize() + totAfterSizeFloored += o.RaiseToFloorSegmentSize(segment.LiveSize()) + } + + if totBeforeSize <= 0 || totAfterSize <= 0 || totAfterSizeFloored <= 0 { + return 0 + } + + // Roughly guess the "balance" of the segments -- whether the + // segments are about the same size. + balance := + float64(o.RaiseToFloorSegmentSize(segments[0].LiveSize())) / + float64(totAfterSizeFloored) + + // Gently favor smaller merges over bigger ones. We don't want to + // make the exponent too large else we end up with poor merges of + // small segments in order to avoid the large merges. + score := balance * math.Pow(float64(totAfterSize), 0.05) + + // Strongly favor merges that reclaim deletes. + nonDelRatio := float64(totAfterSize) / float64(totBeforeSize) + + score *= math.Pow(nonDelRatio, o.ReclaimDeletesWeight) + + return score +} + +// ------------------------------------------ + +// ToBarChart returns an ASCII rendering of the segments and the plan. +// The barMax is the max width of the bars in the bar chart. +func ToBarChart(prefix string, barMax int, segments []Segment, plan *MergePlan) string { + rv := make([]string, 0, len(segments)) + + var maxFullSize int64 + for _, segment := range segments { + if maxFullSize < segment.FullSize() { + maxFullSize = segment.FullSize() + } + } + if maxFullSize < 0 { + maxFullSize = 1 + } + + for _, segment := range segments { + barFull := int(segment.FullSize()) + barLive := int(segment.LiveSize()) + + if maxFullSize > int64(barMax) { + barFull = int(float64(barMax) * float64(barFull) / float64(maxFullSize)) + barLive = int(float64(barMax) * float64(barLive) / float64(maxFullSize)) + } + + barKind := " " + barChar := "." + + if plan != nil { + TASK_LOOP: + for taski, task := range plan.Tasks { + for _, taskSegment := range task.Segments { + if taskSegment == segment { + barKind = "*" + barChar = fmt.Sprintf("%d", taski) + break TASK_LOOP + } + } + } + } + + bar := + strings.Repeat(barChar, barLive)[0:barLive] + + strings.Repeat("x", barFull-barLive)[0:barFull-barLive] + + rv = append(rv, fmt.Sprintf("%s %5d: %5d /%5d - %s %s", prefix, + segment.Id(), + segment.LiveSize(), + segment.FullSize(), + barKind, bar)) + } + + return strings.Join(rv, "\n") +} + +// ValidateMergePlannerOptions validates the merge planner options +func ValidateMergePlannerOptions(options *MergePlanOptions) error { + if options.MaxSegmentSize > MaxSegmentSizeLimit { + return ErrMaxSegmentSizeTooLarge + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/sort.go b/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/sort.go new file mode 100644 index 00000000..d044b8d7 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/mergeplan/sort.go @@ -0,0 +1,28 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mergeplan + +type byLiveSizeDescending []Segment + +func (a byLiveSizeDescending) Len() int { return len(a) } + +func (a byLiveSizeDescending) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +func (a byLiveSizeDescending) Less(i, j int) bool { + if a[i].LiveSize() != a[j].LiveSize() { + return a[i].LiveSize() > a[j].LiveSize() + } + return a[i].Id() < a[j].Id() +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/optimize.go b/vendor/github.com/blevesearch/bleve/index/scorch/optimize.go new file mode 100644 index 00000000..b45fc8b0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/optimize.go @@ -0,0 +1,93 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + + "github.com/RoaringBitmap/roaring" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/scorch/segment/zap" +) + +func (s *IndexSnapshotTermFieldReader) Optimize(kind string, octx index.OptimizableContext) ( + index.OptimizableContext, error) { + if kind != "conjunction" { + return octx, nil + } + + if octx == nil { + octx = &OptimizeTFRConjunction{snapshot: s.snapshot} + } + + o, ok := octx.(*OptimizeTFRConjunction) + if !ok { + return octx, nil + } + + if o.snapshot != s.snapshot { + return nil, fmt.Errorf("tried to optimize across different snapshots") + } + + o.tfrs = append(o.tfrs, s) + + return o, nil +} + +type OptimizeTFRConjunction struct { + snapshot *IndexSnapshot + + tfrs []*IndexSnapshotTermFieldReader +} + +func (o *OptimizeTFRConjunction) Finish() error { + if len(o.tfrs) <= 1 { + return nil + } + + for i := range o.snapshot.segment { + itr0, ok := o.tfrs[0].iterators[i].(*zap.PostingsIterator) + if !ok || itr0.ActualBM == nil { + continue + } + + itr1, ok := o.tfrs[1].iterators[i].(*zap.PostingsIterator) + if !ok || itr1.ActualBM == nil { + continue + } + + bm := roaring.And(itr0.ActualBM, itr1.ActualBM) + + for _, tfr := range o.tfrs[2:] { + itr, ok := tfr.iterators[i].(*zap.PostingsIterator) + if !ok || itr.ActualBM == nil { + continue + } + + bm.And(itr.ActualBM) + } + + for _, tfr := range o.tfrs { + itr, ok := tfr.iterators[i].(*zap.PostingsIterator) + if ok && itr.ActualBM != nil { + itr.ActualBM = bm + itr.Actual = bm.Iterator() + } + } + } + + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/persister.go b/vendor/github.com/blevesearch/bleve/index/scorch/persister.go new file mode 100644 index 00000000..cbc24cdb --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/persister.go @@ -0,0 +1,838 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "bytes" + "encoding/binary" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/index/scorch/segment/zap" + "github.com/boltdb/bolt" +) + +var DefaultChunkFactor uint32 = 1024 + +// Arbitrary number, need to make it configurable. +// Lower values like 10/making persister really slow +// doesn't work well as it is creating more files to +// persist for in next persist iteration and spikes the # FDs. +// Ideal value should let persister also proceed at +// an optimum pace so that the merger can skip +// many intermediate snapshots. +// This needs to be based on empirical data. +// TODO - may need to revisit this approach/value. +var epochDistance = uint64(5) + +type notificationChan chan struct{} + +func (s *Scorch) persisterLoop() { + defer s.asyncTasks.Done() + + var persistWatchers []*epochWatcher + var lastPersistedEpoch, lastMergedEpoch uint64 + var ew *epochWatcher +OUTER: + for { + atomic.AddUint64(&s.stats.TotPersistLoopBeg, 1) + + select { + case <-s.closeCh: + break OUTER + case ew = <-s.persisterNotifier: + persistWatchers = append(persistWatchers, ew) + default: + } + if ew != nil && ew.epoch > lastMergedEpoch { + lastMergedEpoch = ew.epoch + } + lastMergedEpoch, persistWatchers = s.pausePersisterForMergerCatchUp(lastPersistedEpoch, + lastMergedEpoch, persistWatchers) + + var ourSnapshot *IndexSnapshot + var ourPersisted []chan error + + // check to see if there is a new snapshot to persist + s.rootLock.Lock() + if s.root != nil && s.root.epoch > lastPersistedEpoch { + ourSnapshot = s.root + ourSnapshot.AddRef() + ourPersisted = s.rootPersisted + s.rootPersisted = nil + atomic.StoreUint64(&s.iStats.persistSnapshotSize, uint64(ourSnapshot.Size())) + atomic.StoreUint64(&s.iStats.persistEpoch, ourSnapshot.epoch) + } + s.rootLock.Unlock() + + if ourSnapshot != nil { + startTime := time.Now() + + err := s.persistSnapshot(ourSnapshot) + for _, ch := range ourPersisted { + if err != nil { + ch <- err + } + close(ch) + } + if err != nil { + atomic.StoreUint64(&s.iStats.persistEpoch, 0) + if err == ErrClosed { + // index has been closed + _ = ourSnapshot.DecRef() + break OUTER + } + s.fireAsyncError(fmt.Errorf("got err persisting snapshot: %v", err)) + _ = ourSnapshot.DecRef() + atomic.AddUint64(&s.stats.TotPersistLoopErr, 1) + continue OUTER + } + + lastPersistedEpoch = ourSnapshot.epoch + for _, ew := range persistWatchers { + close(ew.notifyCh) + } + + persistWatchers = nil + _ = ourSnapshot.DecRef() + + changed := false + s.rootLock.RLock() + if s.root != nil && s.root.epoch != lastPersistedEpoch { + changed = true + } + s.rootLock.RUnlock() + + s.fireEvent(EventKindPersisterProgress, time.Since(startTime)) + + if changed { + atomic.AddUint64(&s.stats.TotPersistLoopProgress, 1) + continue OUTER + } + } + + // tell the introducer we're waiting for changes + w := &epochWatcher{ + epoch: lastPersistedEpoch, + notifyCh: make(notificationChan, 1), + } + + select { + case <-s.closeCh: + break OUTER + case s.introducerNotifier <- w: + } + + s.removeOldData() // might as well cleanup while waiting + + atomic.AddUint64(&s.stats.TotPersistLoopWait, 1) + + select { + case <-s.closeCh: + break OUTER + case <-w.notifyCh: + // woken up, next loop should pick up work + atomic.AddUint64(&s.stats.TotPersistLoopWaitNotified, 1) + case ew = <-s.persisterNotifier: + // if the watchers are already caught up then let them wait, + // else let them continue to do the catch up + persistWatchers = append(persistWatchers, ew) + } + + atomic.AddUint64(&s.stats.TotPersistLoopEnd, 1) + } +} + +func notifyMergeWatchers(lastPersistedEpoch uint64, + persistWatchers []*epochWatcher) []*epochWatcher { + var watchersNext []*epochWatcher + for _, w := range persistWatchers { + if w.epoch < lastPersistedEpoch { + close(w.notifyCh) + } else { + watchersNext = append(watchersNext, w) + } + } + return watchersNext +} + +func (s *Scorch) pausePersisterForMergerCatchUp(lastPersistedEpoch uint64, lastMergedEpoch uint64, + persistWatchers []*epochWatcher) (uint64, []*epochWatcher) { + + // first, let the watchers proceed if they lag behind + persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) + +OUTER: + // check for slow merger and await until the merger catch up + for lastPersistedEpoch > lastMergedEpoch+epochDistance { + atomic.AddUint64(&s.stats.TotPersisterSlowMergerPause, 1) + + select { + case <-s.closeCh: + break OUTER + case ew := <-s.persisterNotifier: + persistWatchers = append(persistWatchers, ew) + lastMergedEpoch = ew.epoch + } + + atomic.AddUint64(&s.stats.TotPersisterSlowMergerResume, 1) + + // let the watchers proceed if they lag behind + persistWatchers = notifyMergeWatchers(lastPersistedEpoch, persistWatchers) + } + + return lastMergedEpoch, persistWatchers +} + +func (s *Scorch) persistSnapshot(snapshot *IndexSnapshot) error { + persisted, err := s.persistSnapshotMaybeMerge(snapshot) + if err != nil { + return err + } + if persisted { + return nil + } + + return s.persistSnapshotDirect(snapshot) +} + +// DefaultMinSegmentsForInMemoryMerge represents the default number of +// in-memory zap segments that persistSnapshotMaybeMerge() needs to +// see in an IndexSnapshot before it decides to merge and persist +// those segments +var DefaultMinSegmentsForInMemoryMerge = 2 + +// persistSnapshotMaybeMerge examines the snapshot and might merge and +// persist the in-memory zap segments if there are enough of them +func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) ( + bool, error) { + // collect the in-memory zap segments (SegmentBase instances) + var sbs []*zap.SegmentBase + var sbsDrops []*roaring.Bitmap + var sbsIndexes []int + + for i, segmentSnapshot := range snapshot.segment { + if sb, ok := segmentSnapshot.segment.(*zap.SegmentBase); ok { + sbs = append(sbs, sb) + sbsDrops = append(sbsDrops, segmentSnapshot.deleted) + sbsIndexes = append(sbsIndexes, i) + } + } + + if len(sbs) < DefaultMinSegmentsForInMemoryMerge { + return false, nil + } + + newSnapshot, newSegmentID, err := s.mergeSegmentBases( + snapshot, sbs, sbsDrops, sbsIndexes, DefaultChunkFactor) + if err != nil { + return false, err + } + if newSnapshot == nil { + return false, nil + } + + defer func() { + _ = newSnapshot.DecRef() + }() + + mergedSegmentIDs := map[uint64]struct{}{} + for _, idx := range sbsIndexes { + mergedSegmentIDs[snapshot.segment[idx].id] = struct{}{} + } + + // construct a snapshot that's logically equivalent to the input + // snapshot, but with merged segments replaced by the new segment + equiv := &IndexSnapshot{ + parent: snapshot.parent, + segment: make([]*SegmentSnapshot, 0, len(snapshot.segment)), + internal: snapshot.internal, + epoch: snapshot.epoch, + creator: "persistSnapshotMaybeMerge", + } + + // copy to the equiv the segments that weren't replaced + for _, segment := range snapshot.segment { + if _, wasMerged := mergedSegmentIDs[segment.id]; !wasMerged { + equiv.segment = append(equiv.segment, segment) + } + } + + // append to the equiv the new segment + for _, segment := range newSnapshot.segment { + if segment.id == newSegmentID { + equiv.segment = append(equiv.segment, &SegmentSnapshot{ + id: newSegmentID, + segment: segment.segment, + deleted: nil, // nil since merging handled deletions + }) + break + } + } + + err = s.persistSnapshotDirect(equiv) + if err != nil { + return false, err + } + + return true, nil +} + +func (s *Scorch) persistSnapshotDirect(snapshot *IndexSnapshot) (err error) { + // start a write transaction + tx, err := s.rootBolt.Begin(true) + if err != nil { + return err + } + // defer rollback on error + defer func() { + if err != nil { + _ = tx.Rollback() + } + }() + + snapshotsBucket, err := tx.CreateBucketIfNotExists(boltSnapshotsBucket) + if err != nil { + return err + } + newSnapshotKey := segment.EncodeUvarintAscending(nil, snapshot.epoch) + snapshotBucket, err := snapshotsBucket.CreateBucketIfNotExists(newSnapshotKey) + if err != nil { + return err + } + + // persist meta values + metaBucket, err := snapshotBucket.CreateBucketIfNotExists(boltMetaDataKey) + if err != nil { + return err + } + err = metaBucket.Put([]byte("type"), []byte(zap.Type)) + if err != nil { + return err + } + buf := make([]byte, binary.MaxVarintLen32) + binary.BigEndian.PutUint32(buf, zap.Version) + err = metaBucket.Put([]byte("version"), buf) + if err != nil { + return err + } + + // persist internal values + internalBucket, err := snapshotBucket.CreateBucketIfNotExists(boltInternalKey) + if err != nil { + return err + } + // TODO optimize writing these in order? + for k, v := range snapshot.internal { + err = internalBucket.Put([]byte(k), v) + if err != nil { + return err + } + } + + var filenames []string + newSegmentPaths := make(map[uint64]string) + + // first ensure that each segment in this snapshot has been persisted + for _, segmentSnapshot := range snapshot.segment { + snapshotSegmentKey := segment.EncodeUvarintAscending(nil, segmentSnapshot.id) + snapshotSegmentBucket, err := snapshotBucket.CreateBucketIfNotExists(snapshotSegmentKey) + if err != nil { + return err + } + switch seg := segmentSnapshot.segment.(type) { + case *zap.SegmentBase: + // need to persist this to disk + filename := zapFileName(segmentSnapshot.id) + path := s.path + string(os.PathSeparator) + filename + err = zap.PersistSegmentBase(seg, path) + if err != nil { + return fmt.Errorf("error persisting segment: %v", err) + } + newSegmentPaths[segmentSnapshot.id] = path + err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) + if err != nil { + return err + } + filenames = append(filenames, filename) + case *zap.Segment: + path := seg.Path() + filename := strings.TrimPrefix(path, s.path+string(os.PathSeparator)) + err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) + if err != nil { + return err + } + filenames = append(filenames, filename) + default: + return fmt.Errorf("unknown segment type: %T", seg) + } + // store current deleted bits + var roaringBuf bytes.Buffer + if segmentSnapshot.deleted != nil { + _, err = segmentSnapshot.deleted.WriteTo(&roaringBuf) + if err != nil { + return fmt.Errorf("error persisting roaring bytes: %v", err) + } + err = snapshotSegmentBucket.Put(boltDeletedKey, roaringBuf.Bytes()) + if err != nil { + return err + } + } + } + + // we need to swap in a new root only when we've persisted 1 or + // more segments -- whereby the new root would have 1-for-1 + // replacements of in-memory segments with file-based segments + // + // other cases like updates to internal values only, and/or when + // there are only deletions, are already covered and persisted by + // the newly populated boltdb snapshotBucket above + if len(newSegmentPaths) > 0 { + // now try to open all the new snapshots + newSegments := make(map[uint64]segment.Segment) + defer func() { + for _, s := range newSegments { + if s != nil { + // cleanup segments that were opened but not + // swapped into the new root + _ = s.Close() + } + } + }() + for segmentID, path := range newSegmentPaths { + newSegments[segmentID], err = zap.Open(path) + if err != nil { + return fmt.Errorf("error opening new segment at %s, %v", path, err) + } + } + + persist := &persistIntroduction{ + persisted: newSegments, + applied: make(notificationChan), + } + + select { + case <-s.closeCh: + err = ErrClosed + return err + case s.persists <- persist: + } + + select { + case <-s.closeCh: + err = ErrClosed + return err + case <-persist.applied: + } + } + + err = tx.Commit() + if err != nil { + return err + } + + err = s.rootBolt.Sync() + if err != nil { + return err + } + + // allow files to become eligible for removal after commit, such + // as file segments from snapshots that came from the merger + s.rootLock.Lock() + for _, filename := range filenames { + delete(s.ineligibleForRemoval, filename) + } + s.rootLock.Unlock() + + return nil +} + +func zapFileName(epoch uint64) string { + return fmt.Sprintf("%012x.zap", epoch) +} + +// bolt snapshot code + +var boltSnapshotsBucket = []byte{'s'} +var boltPathKey = []byte{'p'} +var boltDeletedKey = []byte{'d'} +var boltInternalKey = []byte{'i'} +var boltMetaDataKey = []byte{'m'} + +func (s *Scorch) loadFromBolt() error { + return s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + foundRoot := false + c := snapshots.Cursor() + for k, _ := c.Last(); k != nil; k, _ = c.Prev() { + _, snapshotEpoch, err := segment.DecodeUvarintAscending(k) + if err != nil { + log.Printf("unable to parse segment epoch %x, continuing", k) + continue + } + if foundRoot { + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + snapshot := snapshots.Bucket(k) + if snapshot == nil { + log.Printf("snapshot key, but bucket missing %x, continuing", k) + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + indexSnapshot, err := s.loadSnapshot(snapshot) + if err != nil { + log.Printf("unable to load snapshot, %v, continuing", err) + s.AddEligibleForRemoval(snapshotEpoch) + continue + } + indexSnapshot.epoch = snapshotEpoch + // set the nextSegmentID + s.nextSegmentID, err = s.maxSegmentIDOnDisk() + if err != nil { + return err + } + s.nextSegmentID++ + s.rootLock.Lock() + s.nextSnapshotEpoch = snapshotEpoch + 1 + if s.root != nil { + _ = s.root.DecRef() + } + s.root = indexSnapshot + s.rootLock.Unlock() + foundRoot = true + } + return nil + }) +} + +// LoadSnapshot loads the segment with the specified epoch +// NOTE: this is currently ONLY intended to be used by the command-line tool +func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) { + err = s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + snapshotKey := segment.EncodeUvarintAscending(nil, epoch) + snapshot := snapshots.Bucket(snapshotKey) + if snapshot == nil { + return fmt.Errorf("snapshot with epoch: %v - doesn't exist", epoch) + } + rv, err = s.loadSnapshot(snapshot) + return err + }) + if err != nil { + return nil, err + } + return rv, nil +} + +func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { + + rv := &IndexSnapshot{ + parent: s, + internal: make(map[string][]byte), + refs: 1, + creator: "loadSnapshot", + } + var running uint64 + c := snapshot.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + if k[0] == boltInternalKey[0] { + internalBucket := snapshot.Bucket(k) + err := internalBucket.ForEach(func(key []byte, val []byte) error { + copiedVal := append([]byte(nil), val...) + rv.internal[string(key)] = copiedVal + return nil + }) + if err != nil { + _ = rv.DecRef() + return nil, err + } + } else if k[0] != boltMetaDataKey[0] { + segmentBucket := snapshot.Bucket(k) + if segmentBucket == nil { + _ = rv.DecRef() + return nil, fmt.Errorf("segment key, but bucket missing % x", k) + } + segmentSnapshot, err := s.loadSegment(segmentBucket) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("failed to load segment: %v", err) + } + _, segmentSnapshot.id, err = segment.DecodeUvarintAscending(k) + if err != nil { + _ = rv.DecRef() + return nil, fmt.Errorf("failed to decode segment id: %v", err) + } + rv.segment = append(rv.segment, segmentSnapshot) + rv.offsets = append(rv.offsets, running) + running += segmentSnapshot.segment.Count() + } + } + return rv, nil +} + +func (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, error) { + pathBytes := segmentBucket.Get(boltPathKey) + if pathBytes == nil { + return nil, fmt.Errorf("segment path missing") + } + segmentPath := s.path + string(os.PathSeparator) + string(pathBytes) + segment, err := zap.Open(segmentPath) + if err != nil { + return nil, fmt.Errorf("error opening bolt segment: %v", err) + } + + rv := &SegmentSnapshot{ + segment: segment, + cachedDocs: &cachedDocs{cache: nil}, + } + deletedBytes := segmentBucket.Get(boltDeletedKey) + if deletedBytes != nil { + deletedBitmap := roaring.NewBitmap() + r := bytes.NewReader(deletedBytes) + _, err := deletedBitmap.ReadFrom(r) + if err != nil { + _ = segment.Close() + return nil, fmt.Errorf("error reading deleted bytes: %v", err) + } + if !deletedBitmap.IsEmpty() { + rv.deleted = deletedBitmap + } + } + + return rv, nil +} + +type uint64Descending []uint64 + +func (p uint64Descending) Len() int { return len(p) } +func (p uint64Descending) Less(i, j int) bool { return p[i] > p[j] } +func (p uint64Descending) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func (s *Scorch) removeOldData() { + removed, err := s.removeOldBoltSnapshots() + if err != nil { + s.fireAsyncError(fmt.Errorf("got err removing old bolt snapshots: %v", err)) + } + + if removed > 0 { + err = s.removeOldZapFiles() + if err != nil { + s.fireAsyncError(fmt.Errorf("got err removing old zap files: %v", err)) + } + } +} + +// NumSnapshotsToKeep represents how many recent, old snapshots to +// keep around per Scorch instance. Useful for apps that require +// rollback'ability. +var NumSnapshotsToKeep = 1 + +// Removes enough snapshots from the rootBolt so that the +// s.eligibleForRemoval stays under the NumSnapshotsToKeep policy. +func (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) { + persistedEpochs, err := s.RootBoltSnapshotEpochs() + if err != nil { + return 0, err + } + + if len(persistedEpochs) <= s.numSnapshotsToKeep { + // we need to keep everything + return 0, nil + } + + // make a map of epochs to protect from deletion + protectedEpochs := make(map[uint64]struct{}, s.numSnapshotsToKeep) + for _, epoch := range persistedEpochs[0:s.numSnapshotsToKeep] { + protectedEpochs[epoch] = struct{}{} + } + + var epochsToRemove []uint64 + var newEligible []uint64 + s.rootLock.Lock() + for _, epoch := range s.eligibleForRemoval { + if _, ok := protectedEpochs[epoch]; ok { + // protected + newEligible = append(newEligible, epoch) + } else { + epochsToRemove = append(epochsToRemove, epoch) + } + } + s.eligibleForRemoval = newEligible + s.rootLock.Unlock() + + if len(epochsToRemove) <= 0 { + return 0, nil + } + + tx, err := s.rootBolt.Begin(true) + if err != nil { + return 0, err + } + defer func() { + if err == nil { + err = tx.Commit() + } else { + _ = tx.Rollback() + } + if err == nil { + err = s.rootBolt.Sync() + } + }() + + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return 0, nil + } + + for _, epochToRemove := range epochsToRemove { + k := segment.EncodeUvarintAscending(nil, epochToRemove) + err = snapshots.DeleteBucket(k) + if err == bolt.ErrBucketNotFound { + err = nil + } + if err == nil { + numRemoved++ + } + } + + return numRemoved, err +} + +func (s *Scorch) maxSegmentIDOnDisk() (uint64, error) { + currFileInfos, err := ioutil.ReadDir(s.path) + if err != nil { + return 0, err + } + + var rv uint64 + for _, finfo := range currFileInfos { + fname := finfo.Name() + if filepath.Ext(fname) == ".zap" { + prefix := strings.TrimSuffix(fname, ".zap") + id, err2 := strconv.ParseUint(prefix, 16, 64) + if err2 != nil { + return 0, err2 + } + if id > rv { + rv = id + } + } + } + return rv, err +} + +// Removes any *.zap files which aren't listed in the rootBolt. +func (s *Scorch) removeOldZapFiles() error { + liveFileNames, err := s.loadZapFileNames() + if err != nil { + return err + } + + currFileInfos, err := ioutil.ReadDir(s.path) + if err != nil { + return err + } + + s.rootLock.RLock() + + for _, finfo := range currFileInfos { + fname := finfo.Name() + if filepath.Ext(fname) == ".zap" { + if _, exists := liveFileNames[fname]; !exists && !s.ineligibleForRemoval[fname] { + err := os.Remove(s.path + string(os.PathSeparator) + fname) + if err != nil { + log.Printf("got err removing file: %s, err: %v", fname, err) + } + } + } + } + + s.rootLock.RUnlock() + + return nil +} + +func (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) { + var rv []uint64 + err := s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + for sk, _ := sc.Last(); sk != nil; sk, _ = sc.Prev() { + _, snapshotEpoch, err := segment.DecodeUvarintAscending(sk) + if err != nil { + continue + } + rv = append(rv, snapshotEpoch) + } + return nil + }) + return rv, err +} + +// Returns the *.zap file names that are listed in the rootBolt. +func (s *Scorch) loadZapFileNames() (map[string]struct{}, error) { + rv := map[string]struct{}{} + err := s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil + } + sc := snapshots.Cursor() + for sk, _ := sc.First(); sk != nil; sk, _ = sc.Next() { + snapshot := snapshots.Bucket(sk) + if snapshot == nil { + continue + } + segc := snapshot.Cursor() + for segk, _ := segc.First(); segk != nil; segk, _ = segc.Next() { + if segk[0] == boltInternalKey[0] { + continue + } + segmentBucket := snapshot.Bucket(segk) + if segmentBucket == nil { + continue + } + pathBytes := segmentBucket.Get(boltPathKey) + if pathBytes == nil { + continue + } + pathString := string(pathBytes) + rv[string(pathString)] = struct{}{} + } + } + return nil + }) + + return rv, err +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go b/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go new file mode 100644 index 00000000..31d31642 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go @@ -0,0 +1,573 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/index/scorch/segment/zap" + "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/registry" + "github.com/boltdb/bolt" +) + +const Name = "scorch" + +const Version uint8 = 2 + +var ErrClosed = fmt.Errorf("scorch closed") + +type Scorch struct { + readOnly bool + version uint8 + config map[string]interface{} + analysisQueue *index.AnalysisQueue + stats Stats + nextSegmentID uint64 + path string + + unsafeBatch bool + + rootLock sync.RWMutex + root *IndexSnapshot // holds 1 ref-count on the root + rootPersisted []chan error // closed when root is persisted + nextSnapshotEpoch uint64 + eligibleForRemoval []uint64 // Index snapshot epochs that are safe to GC. + ineligibleForRemoval map[string]bool // Filenames that should not be GC'ed yet. + + numSnapshotsToKeep int + closeCh chan struct{} + introductions chan *segmentIntroduction + persists chan *persistIntroduction + merges chan *segmentMerge + introducerNotifier chan *epochWatcher + revertToSnapshots chan *snapshotReversion + persisterNotifier chan *epochWatcher + rootBolt *bolt.DB + asyncTasks sync.WaitGroup + + onEvent func(event Event) + onAsyncError func(err error) + + iStats internalStats +} + +type internalStats struct { + persistEpoch uint64 + persistSnapshotSize uint64 + mergeEpoch uint64 + mergeSnapshotSize uint64 + newSegBufBytesAdded uint64 + newSegBufBytesRemoved uint64 + analysisBytesAdded uint64 + analysisBytesRemoved uint64 +} + +func NewScorch(storeName string, + config map[string]interface{}, + analysisQueue *index.AnalysisQueue) (index.Index, error) { + rv := &Scorch{ + version: Version, + config: config, + analysisQueue: analysisQueue, + nextSnapshotEpoch: 1, + closeCh: make(chan struct{}), + ineligibleForRemoval: map[string]bool{}, + } + rv.root = &IndexSnapshot{parent: rv, refs: 1, creator: "NewScorch"} + ro, ok := config["read_only"].(bool) + if ok { + rv.readOnly = ro + } + ub, ok := config["unsafe_batch"].(bool) + if ok { + rv.unsafeBatch = ub + } + ecbName, ok := config["eventCallbackName"].(string) + if ok { + rv.onEvent = RegistryEventCallbacks[ecbName] + } + aecbName, ok := config["asyncErrorCallbackName"].(string) + if ok { + rv.onAsyncError = RegistryAsyncErrorCallbacks[aecbName] + } + return rv, nil +} + +func (s *Scorch) fireEvent(kind EventKind, dur time.Duration) { + if s.onEvent != nil { + s.onEvent(Event{Kind: kind, Scorch: s, Duration: dur}) + } +} + +func (s *Scorch) fireAsyncError(err error) { + if s.onAsyncError != nil { + s.onAsyncError(err) + } + atomic.AddUint64(&s.stats.TotOnErrors, 1) +} + +func (s *Scorch) Open() error { + err := s.openBolt() + if err != nil { + return err + } + + s.asyncTasks.Add(1) + go s.mainLoop() + + if !s.readOnly && s.path != "" { + s.asyncTasks.Add(1) + go s.persisterLoop() + s.asyncTasks.Add(1) + go s.mergerLoop() + } + + return nil +} + +func (s *Scorch) openBolt() error { + var ok bool + s.path, ok = s.config["path"].(string) + if !ok { + return fmt.Errorf("must specify path") + } + if s.path == "" { + s.unsafeBatch = true + } + + var rootBoltOpt *bolt.Options + if s.readOnly { + rootBoltOpt = &bolt.Options{ + ReadOnly: true, + } + } else { + if s.path != "" { + err := os.MkdirAll(s.path, 0700) + if err != nil { + return err + } + } + } + + rootBoltPath := s.path + string(os.PathSeparator) + "root.bolt" + var err error + if s.path != "" { + s.rootBolt, err = bolt.Open(rootBoltPath, 0600, rootBoltOpt) + if err != nil { + return err + } + + // now see if there is any existing state to load + err = s.loadFromBolt() + if err != nil { + _ = s.Close() + return err + } + } + + s.introductions = make(chan *segmentIntroduction) + s.persists = make(chan *persistIntroduction) + s.merges = make(chan *segmentMerge) + s.introducerNotifier = make(chan *epochWatcher, 1) + s.revertToSnapshots = make(chan *snapshotReversion) + s.persisterNotifier = make(chan *epochWatcher, 1) + + if !s.readOnly && s.path != "" { + err := s.removeOldZapFiles() // Before persister or merger create any new files. + if err != nil { + _ = s.Close() + return err + } + } + + s.numSnapshotsToKeep = NumSnapshotsToKeep + if v, ok := s.config["numSnapshotsToKeep"]; ok { + var t int + if t, err = parseToInteger(v); err != nil { + return fmt.Errorf("numSnapshotsToKeep parse err: %v", err) + } + if t > 0 { + s.numSnapshotsToKeep = t + } + } + + return nil +} + +func (s *Scorch) Close() (err error) { + startTime := time.Now() + defer func() { + s.fireEvent(EventKindClose, time.Since(startTime)) + }() + + s.fireEvent(EventKindCloseStart, 0) + + // signal to async tasks we want to close + close(s.closeCh) + // wait for them to close + s.asyncTasks.Wait() + // now close the root bolt + if s.rootBolt != nil { + err = s.rootBolt.Close() + s.rootLock.Lock() + if s.root != nil { + _ = s.root.DecRef() + } + s.root = nil + s.rootLock.Unlock() + } + + return +} + +func (s *Scorch) Update(doc *document.Document) error { + b := index.NewBatch() + b.Update(doc) + return s.Batch(b) +} + +func (s *Scorch) Delete(id string) error { + b := index.NewBatch() + b.Delete(id) + return s.Batch(b) +} + +// Batch applices a batch of changes to the index atomically +func (s *Scorch) Batch(batch *index.Batch) (err error) { + start := time.Now() + + defer func() { + s.fireEvent(EventKindBatchIntroduction, time.Since(start)) + }() + + resultChan := make(chan *index.AnalysisResult, len(batch.IndexOps)) + + var numUpdates uint64 + var numDeletes uint64 + var numPlainTextBytes uint64 + var ids []string + for docID, doc := range batch.IndexOps { + if doc != nil { + // insert _id field + doc.AddField(document.NewTextFieldCustom("_id", nil, []byte(doc.ID), document.IndexField|document.StoreField, nil)) + numUpdates++ + numPlainTextBytes += doc.NumPlainTextBytes() + } else { + numDeletes++ + } + ids = append(ids, docID) + } + + // FIXME could sort ids list concurrent with analysis? + + go func() { + for _, doc := range batch.IndexOps { + if doc != nil { + aw := index.NewAnalysisWork(s, doc, resultChan) + // put the work on the queue + s.analysisQueue.Queue(aw) + } + } + }() + + // wait for analysis result + analysisResults := make([]*index.AnalysisResult, int(numUpdates)) + var itemsDeQueued uint64 + var totalAnalysisSize int + for itemsDeQueued < numUpdates { + result := <-resultChan + resultSize := result.Size() + atomic.AddUint64(&s.iStats.analysisBytesAdded, uint64(resultSize)) + totalAnalysisSize += resultSize + analysisResults[itemsDeQueued] = result + itemsDeQueued++ + } + close(resultChan) + defer atomic.AddUint64(&s.iStats.analysisBytesRemoved, uint64(totalAnalysisSize)) + + atomic.AddUint64(&s.stats.TotAnalysisTime, uint64(time.Since(start))) + + indexStart := time.Now() + + // notify handlers that we're about to introduce a segment + s.fireEvent(EventKindBatchIntroductionStart, 0) + + var newSegment segment.Segment + var bufBytes uint64 + if len(analysisResults) > 0 { + newSegment, bufBytes, err = zap.AnalysisResultsToSegmentBase(analysisResults, DefaultChunkFactor) + if err != nil { + return err + } + atomic.AddUint64(&s.iStats.newSegBufBytesAdded, bufBytes) + } else { + atomic.AddUint64(&s.stats.TotBatchesEmpty, 1) + } + + err = s.prepareSegment(newSegment, ids, batch.InternalOps) + if err != nil { + if newSegment != nil { + _ = newSegment.Close() + } + atomic.AddUint64(&s.stats.TotOnErrors, 1) + } else { + atomic.AddUint64(&s.stats.TotUpdates, numUpdates) + atomic.AddUint64(&s.stats.TotDeletes, numDeletes) + atomic.AddUint64(&s.stats.TotBatches, 1) + atomic.AddUint64(&s.stats.TotIndexedPlainTextBytes, numPlainTextBytes) + } + + atomic.AddUint64(&s.iStats.newSegBufBytesRemoved, bufBytes) + atomic.AddUint64(&s.stats.TotIndexTime, uint64(time.Since(indexStart))) + + return err +} + +func (s *Scorch) prepareSegment(newSegment segment.Segment, ids []string, + internalOps map[string][]byte) error { + + // new introduction + introduction := &segmentIntroduction{ + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: newSegment, + ids: ids, + obsoletes: make(map[uint64]*roaring.Bitmap), + internal: internalOps, + applied: make(chan error), + } + + if !s.unsafeBatch { + introduction.persisted = make(chan error, 1) + } + + // optimistically prepare obsoletes outside of rootLock + s.rootLock.RLock() + root := s.root + root.AddRef() + s.rootLock.RUnlock() + + for _, seg := range root.segment { + delta, err := seg.segment.DocNumbers(ids) + if err != nil { + return err + } + introduction.obsoletes[seg.id] = delta + } + + _ = root.DecRef() + + introStartTime := time.Now() + + s.introductions <- introduction + + // block until this segment is applied + err := <-introduction.applied + if err != nil { + return err + } + + if introduction.persisted != nil { + err = <-introduction.persisted + } + + introTime := uint64(time.Since(introStartTime)) + atomic.AddUint64(&s.stats.TotBatchIntroTime, introTime) + if atomic.LoadUint64(&s.stats.MaxBatchIntroTime) < introTime { + atomic.StoreUint64(&s.stats.MaxBatchIntroTime, introTime) + } + + return err +} + +func (s *Scorch) SetInternal(key, val []byte) error { + b := index.NewBatch() + b.SetInternal(key, val) + return s.Batch(b) +} + +func (s *Scorch) DeleteInternal(key []byte) error { + b := index.NewBatch() + b.DeleteInternal(key) + return s.Batch(b) +} + +// Reader returns a low-level accessor on the index data. Close it to +// release associated resources. +func (s *Scorch) Reader() (index.IndexReader, error) { + return s.currentSnapshot(), nil +} + +func (s *Scorch) currentSnapshot() *IndexSnapshot { + s.rootLock.RLock() + rv := s.root + rv.AddRef() + s.rootLock.RUnlock() + return rv +} + +func (s *Scorch) Stats() json.Marshaler { + return &s.stats +} +func (s *Scorch) StatsMap() map[string]interface{} { + m := s.stats.ToMap() + + if s.path != "" { + finfos, err := ioutil.ReadDir(s.path) + if err == nil { + var numFilesOnDisk, numBytesUsedDisk uint64 + for _, finfo := range finfos { + if !finfo.IsDir() { + numBytesUsedDisk += uint64(finfo.Size()) + numFilesOnDisk++ + } + } + + m["CurOnDiskBytes"] = numBytesUsedDisk + m["CurOnDiskFiles"] = numFilesOnDisk + } + } + + // TODO: consider one day removing these backwards compatible + // names for apps using the old names + m["updates"] = m["TotUpdates"] + m["deletes"] = m["TotDeletes"] + m["batches"] = m["TotBatches"] + m["errors"] = m["TotOnErrors"] + m["analysis_time"] = m["TotAnalysisTime"] + m["index_time"] = m["TotIndexTime"] + m["term_searchers_started"] = m["TotTermSearchersStarted"] + m["term_searchers_finished"] = m["TotTermSearchersFinished"] + m["num_plain_text_bytes_indexed"] = m["TotIndexedPlainTextBytes"] + m["num_items_introduced"] = m["TotIntroducedItems"] + m["num_items_persisted"] = m["TotPersistedItems"] + m["num_bytes_used_disk"] = m["CurOnDiskBytes"] + m["num_files_on_disk"] = m["CurOnDiskFiles"] + m["total_compaction_written_bytes"] = m["TotFileMergeWrittenBytes"] + + return m +} + +func (s *Scorch) Analyze(d *document.Document) *index.AnalysisResult { + rv := &index.AnalysisResult{ + Document: d, + Analyzed: make([]analysis.TokenFrequencies, len(d.Fields)+len(d.CompositeFields)), + Length: make([]int, len(d.Fields)+len(d.CompositeFields)), + } + + for i, field := range d.Fields { + if field.Options().IsIndexed() { + fieldLength, tokenFreqs := field.Analyze() + rv.Analyzed[i] = tokenFreqs + rv.Length[i] = fieldLength + + if len(d.CompositeFields) > 0 { + // see if any of the composite fields need this + for _, compositeField := range d.CompositeFields { + compositeField.Compose(field.Name(), fieldLength, tokenFreqs) + } + } + } + } + + return rv +} + +func (s *Scorch) Advanced() (store.KVStore, error) { + return nil, nil +} + +func (s *Scorch) AddEligibleForRemoval(epoch uint64) { + s.rootLock.Lock() + if s.root == nil || s.root.epoch != epoch { + s.eligibleForRemoval = append(s.eligibleForRemoval, epoch) + } + s.rootLock.Unlock() +} + +func (s *Scorch) MemoryUsed() uint64 { + indexSnapshot := s.currentSnapshot() + defer func() { + _ = indexSnapshot.Close() + }() + + // Account for current root snapshot overhead + memUsed := uint64(indexSnapshot.Size()) + + // Account for snapshot that the persister may be working on + persistEpoch := atomic.LoadUint64(&s.iStats.persistEpoch) + persistSnapshotSize := atomic.LoadUint64(&s.iStats.persistSnapshotSize) + if persistEpoch != 0 && indexSnapshot.epoch > persistEpoch { + // the snapshot that the persister is working on isn't the same as + // the current snapshot + memUsed += persistSnapshotSize + } + + // Account for snapshot that the merger may be working on + mergeEpoch := atomic.LoadUint64(&s.iStats.mergeEpoch) + mergeSnapshotSize := atomic.LoadUint64(&s.iStats.mergeSnapshotSize) + if mergeEpoch != 0 && indexSnapshot.epoch > mergeEpoch { + // the snapshot that the merger is working on isn't the same as + // the current snapshot + memUsed += mergeSnapshotSize + } + + memUsed += (atomic.LoadUint64(&s.iStats.newSegBufBytesAdded) - + atomic.LoadUint64(&s.iStats.newSegBufBytesRemoved)) + + memUsed += (atomic.LoadUint64(&s.iStats.analysisBytesAdded) - + atomic.LoadUint64(&s.iStats.analysisBytesRemoved)) + + return memUsed +} + +func (s *Scorch) markIneligibleForRemoval(filename string) { + s.rootLock.Lock() + s.ineligibleForRemoval[filename] = true + s.rootLock.Unlock() +} + +func (s *Scorch) unmarkIneligibleForRemoval(filename string) { + s.rootLock.Lock() + delete(s.ineligibleForRemoval, filename) + s.rootLock.Unlock() +} + +func init() { + registry.RegisterIndexType(Name, NewScorch) +} + +func parseToInteger(i interface{}) (int, error) { + switch v := i.(type) { + case float64: + return int(v), nil + case int: + return v, nil + + default: + return 0, fmt.Errorf("expects int or float64 value") + } +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go new file mode 100644 index 00000000..0489c821 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/empty.go @@ -0,0 +1,130 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package segment + +import ( + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index" +) + +type EmptySegment struct{} + +func (e *EmptySegment) Dictionary(field string) (TermDictionary, error) { + return &EmptyDictionary{}, nil +} + +func (e *EmptySegment) VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error { + return nil +} + +func (e *EmptySegment) DocID(num uint64) ([]byte, error) { + return nil, nil +} + +func (e *EmptySegment) Count() uint64 { + return 0 +} + +func (e *EmptySegment) DocNumbers([]string) (*roaring.Bitmap, error) { + r := roaring.NewBitmap() + return r, nil +} + +func (e *EmptySegment) Fields() []string { + return []string{} +} + +func (e *EmptySegment) Close() error { + return nil +} + +func (e *EmptySegment) Size() uint64 { + return 0 +} + +func (e *EmptySegment) AddRef() { +} + +func (e *EmptySegment) DecRef() error { + return nil +} + +type EmptyDictionary struct{} + +func (e *EmptyDictionary) PostingsList(term []byte, + except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) { + return &EmptyPostingsList{}, nil +} + +func (e *EmptyDictionary) Iterator() DictionaryIterator { + return &EmptyDictionaryIterator{} +} + +func (e *EmptyDictionary) PrefixIterator(prefix string) DictionaryIterator { + return &EmptyDictionaryIterator{} +} + +func (e *EmptyDictionary) RangeIterator(start, end string) DictionaryIterator { + return &EmptyDictionaryIterator{} +} + +func (e *EmptyDictionary) RegexpIterator(start string) DictionaryIterator { + return &EmptyDictionaryIterator{} +} + +func (e *EmptyDictionary) FuzzyIterator(term string, + fuzziness int) DictionaryIterator { + return &EmptyDictionaryIterator{} +} + +func (e *EmptyDictionary) OnlyIterator(onlyTerms [][]byte, + includeCount bool) DictionaryIterator { + return &EmptyDictionaryIterator{} +} + +type EmptyDictionaryIterator struct{} + +func (e *EmptyDictionaryIterator) Next() (*index.DictEntry, error) { + return nil, nil +} + +func (e *EmptyPostingsIterator) Advance(uint64) (Posting, error) { + return nil, nil +} + +type EmptyPostingsList struct{} + +func (e *EmptyPostingsList) Iterator(includeFreq, includeNorm, includeLocations bool, + prealloc PostingsIterator) PostingsIterator { + return &EmptyPostingsIterator{} +} + +func (e *EmptyPostingsList) Size() int { + return 0 +} + +func (e *EmptyPostingsList) Count() uint64 { + return 0 +} + +type EmptyPostingsIterator struct{} + +func (e *EmptyPostingsIterator) Next() (Posting, error) { + return nil, nil +} + +func (e *EmptyPostingsIterator) Size() int { + return 0 +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go new file mode 100644 index 00000000..a4836ebf --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/int.go @@ -0,0 +1,94 @@ +// Copyright 2014 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +// This code originated from: +// https://github.com/cockroachdb/cockroach/blob/2dd65dde5d90c157f4b93f92502ca1063b904e1d/pkg/util/encoding/encoding.go + +// Modified to not use pkg/errors + +package segment + +import "fmt" + +const ( + MaxVarintSize = 9 + + // IntMin is chosen such that the range of int tags does not overlap the + // ascii character set that is frequently used in testing. + IntMin = 0x80 // 128 + intMaxWidth = 8 + intZero = IntMin + intMaxWidth // 136 + intSmall = IntMax - intZero - intMaxWidth // 109 + // IntMax is the maximum int tag value. + IntMax = 0xfd // 253 +) + +// EncodeUvarintAscending encodes the uint64 value using a variable length +// (length-prefixed) representation. The length is encoded as a single +// byte indicating the number of encoded bytes (-8) to follow. See +// EncodeVarintAscending for rationale. The encoded bytes are appended to the +// supplied buffer and the final buffer is returned. +func EncodeUvarintAscending(b []byte, v uint64) []byte { + switch { + case v <= intSmall: + return append(b, intZero+byte(v)) + case v <= 0xff: + return append(b, IntMax-7, byte(v)) + case v <= 0xffff: + return append(b, IntMax-6, byte(v>>8), byte(v)) + case v <= 0xffffff: + return append(b, IntMax-5, byte(v>>16), byte(v>>8), byte(v)) + case v <= 0xffffffff: + return append(b, IntMax-4, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + case v <= 0xffffffffff: + return append(b, IntMax-3, byte(v>>32), byte(v>>24), byte(v>>16), byte(v>>8), + byte(v)) + case v <= 0xffffffffffff: + return append(b, IntMax-2, byte(v>>40), byte(v>>32), byte(v>>24), byte(v>>16), + byte(v>>8), byte(v)) + case v <= 0xffffffffffffff: + return append(b, IntMax-1, byte(v>>48), byte(v>>40), byte(v>>32), byte(v>>24), + byte(v>>16), byte(v>>8), byte(v)) + default: + return append(b, IntMax, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), + byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + } +} + +// DecodeUvarintAscending decodes a varint encoded uint64 from the input +// buffer. The remainder of the input buffer and the decoded uint64 +// are returned. +func DecodeUvarintAscending(b []byte) ([]byte, uint64, error) { + if len(b) == 0 { + return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value") + } + length := int(b[0]) - intZero + b = b[1:] // skip length byte + if length <= intSmall { + return b, uint64(length), nil + } + length -= intSmall + if length < 0 || length > 8 { + return nil, 0, fmt.Errorf("invalid uvarint length of %d", length) + } else if len(b) < length { + return nil, 0, fmt.Errorf("insufficient bytes to decode uvarint value: %q", b) + } + var v uint64 + // It is faster to range over the elements in a slice than to index + // into the slice on each loop iteration. + for _, t := range b[:length] { + v = (v << 8) | uint64(t) + } + return b[length:], v, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go new file mode 100644 index 00000000..3fc31599 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/segment.go @@ -0,0 +1,126 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package segment + +import ( + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index" +) + +// DocumentFieldValueVisitor defines a callback to be visited for each +// stored field value. The return value determines if the visitor +// should keep going. Returning true continues visiting, false stops. +type DocumentFieldValueVisitor func(field string, typ byte, value []byte, pos []uint64) bool + +type Segment interface { + Dictionary(field string) (TermDictionary, error) + + VisitDocument(num uint64, visitor DocumentFieldValueVisitor) error + + DocID(num uint64) ([]byte, error) + + Count() uint64 + + DocNumbers([]string) (*roaring.Bitmap, error) + + Fields() []string + + Close() error + + Size() int + + AddRef() + DecRef() error +} + +type TermDictionary interface { + PostingsList(term []byte, except *roaring.Bitmap, prealloc PostingsList) (PostingsList, error) + + Iterator() DictionaryIterator + PrefixIterator(prefix string) DictionaryIterator + RangeIterator(start, end string) DictionaryIterator + RegexpIterator(regex string) DictionaryIterator + FuzzyIterator(term string, fuzziness int) DictionaryIterator + OnlyIterator(onlyTerms [][]byte, includeCount bool) DictionaryIterator +} + +type DictionaryIterator interface { + Next() (*index.DictEntry, error) +} + +type PostingsList interface { + Iterator(includeFreq, includeNorm, includeLocations bool, prealloc PostingsIterator) PostingsIterator + + Size() int + + Count() uint64 + + // NOTE deferred for future work + + // And(other PostingsList) PostingsList + // Or(other PostingsList) PostingsList +} + +type PostingsIterator interface { + // The caller is responsible for copying whatever it needs from + // the returned Posting instance before calling Next(), as some + // implementations may return a shared instance to reduce memory + // allocations. + Next() (Posting, error) + + // Advance will return the posting with the specified doc number + // or if there is no such posting, the next posting. + // Callers MUST NOT attempt to pass a docNum that is less than or + // equal to the currently visited posting doc Num. + Advance(docNum uint64) (Posting, error) + + Size() int +} + +type Posting interface { + Number() uint64 + + Frequency() uint64 + Norm() float64 + + Locations() []Location + + Size() int +} + +type Location interface { + Field() string + Start() uint64 + End() uint64 + Pos() uint64 + ArrayPositions() []uint64 + Size() int +} + +// DocumentFieldTermVisitable is implemented by various scorch segment +// implementations with persistence for the un inverting of the +// postings or other indexed values. +type DocumentFieldTermVisitable interface { + VisitDocumentFieldTerms(localDocNum uint64, fields []string, + visitor index.DocumentFieldTermVisitor, optional DocVisitState) (DocVisitState, error) + + // VisitableDocValueFields implementation should return + // the list of fields which are document value persisted and + // therefore visitable by the above VisitDocumentFieldTerms method. + VisitableDocValueFields() ([]string, error) +} + +type DocVisitState interface { +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/README.md b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/README.md new file mode 100644 index 00000000..41f5902a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/README.md @@ -0,0 +1,167 @@ +# zap file format + +The file is written in the reverse order that we typically access data. This helps us write in one pass since later sections of the file require file offsets of things we've already written. + +Current usage: + +- mmap the entire file +- crc-32 bytes and version are in fixed position at end of the file +- reading remainder of footer could be version specific +- remainder of footer gives us: + - 3 important offsets (docValue , fields index and stored data index) + - 2 important values (number of docs and chunk factor) +- field data is processed once and memoized onto the heap so that we never have to go back to disk for it +- access to stored data by doc number means first navigating to the stored data index, then accessing a fixed position offset into that slice, which gives us the actual address of the data. the first bytes of that section tell us the size of data so that we know where it ends. +- access to all other indexed data follows the following pattern: + - first know the field name -> convert to id + - next navigate to term dictionary for that field + - some operations stop here and do dictionary ops + - next use dictionary to navigate to posting list for a specific term + - walk posting list + - if necessary, walk posting details as we go + - if location info is desired, consult location bitmap to see if it is there + +## stored fields section + +- for each document + - preparation phase: + - produce a slice of metadata bytes and data bytes + - produce these slices in field id order + - field value is appended to the data slice + - metadata slice is varint encoded with the following values for each field value + - field id (uint16) + - field type (byte) + - field value start offset in uncompressed data slice (uint64) + - field value length (uint64) + - field number of array positions (uint64) + - one additional value for each array position (uint64) + - compress the data slice using snappy + - file writing phase: + - remember the start offset for this document + - write out meta data length (varint uint64) + - write out compressed data length (varint uint64) + - write out the metadata bytes + - write out the compressed data bytes + +## stored fields idx + +- for each document + - write start offset (remembered from previous section) of stored data (big endian uint64) + +With this index and a known document number, we have direct access to all the stored field data. + +## posting details (freq/norm) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode term frequency (uint64) + - encode norm factor (float32) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## posting details (location) section + +- for each posting list + - produce a slice containing multiple consecutive chunks (each chunk is varint stream) + - produce a slice remembering offsets of where each chunk starts + - preparation phase: + - for each hit in the posting list + - if this hit is in next chunk close out encoding of last chunk and record offset start of next + - encode field (uint16) + - encode field pos (uint64) + - encode field start (uint64) + - encode field end (uint64) + - encode number of array positions to follow (uint64) + - encode each array position (each uint64) + - file writing phase: + - remember start position for this posting list details + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +If you know the doc number you're interested in, this format lets you jump to the correct chunk (docNum/chunkFactor) directly and then seek within that chunk until you find it. + +## bitmaps of hits with location info + +- for each posting list + - preparation phase: + - encode roaring bitmap (inidicating which hits have location details indexed) posting list to bytes (so we know the length) + - file writing phase: + - remember the start position for this bitmap + - write length of encoded roaring bitmap + - write the serialized roaring bitmap data + +## postings list section + +- for each posting list + - preparation phase: + - encode roaring bitmap posting list to bytes (so we know the length) + - file writing phase: + - remember the start position for this posting list + - write freq/norm details offset (remembered from previous, as varint uint64) + - write location details offset (remembered from previous, as varint uint64) + - write location bitmap offset (remembered from pervious, as varint uint64) + - write length of encoded roaring bitmap + - write the serialized roaring bitmap data + +## dictionary + +- for each field + - preparation phase: + - encode vellum FST with dictionary data pointing to file offset of posting list (remembered from previous) + - file writing phase: + - remember the start position of this persistDictionary + - write length of vellum data (varint uint64) + - write out vellum data + +## fields section + +- for each field + - file writing phase: + - remember start offset for each field + - write dictionary address (remembered from previous) (varint uint64) + - write length of field name (varint uint64) + - write field name bytes + +## fields idx + +- for each field + - file writing phase: + - write big endian uint64 of start offset for each field + +NOTE: currently we don't know or record the length of this fields index. Instead we rely on the fact that we know it immediately precedes a footer of known size. + +## fields DocValue + +- for each field + - preparation phase: + - produce a slice containing multiple consecutive chunks, where each chunk is composed of a meta section followed by compressed columnar field data + - produce a slice remembering the length of each chunk + - file writing phase: + - remember the start position of this first field DocValue offset in the footer + - write out number of chunks that follow (varint uint64) + - write out length of each chunk (each a varint uint64) + - write out the byte slice containing all the chunk data + +NOTE: currently the meta header inside each chunk gives clue to the location offsets and size of the data pertaining to a given docID and any +read operation leverage that meta information to extract the document specific data from the file. + +## footer + +- file writing phase + - write number of docs (big endian uint64) + - write stored field index location (big endian uint64) + - write field index location (big endian uint64) + - write field docValue location (big endian uint64) + - write out chunk factor (big endian uint32) + - write out version (big endian uint32) + - write out file CRC of everything preceding this (big endian uint32) diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/build.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/build.go new file mode 100644 index 00000000..91bfd4e2 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/build.go @@ -0,0 +1,149 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "math" + "os" +) + +const Version uint32 = 11 + +const Type string = "zap" + +const fieldNotUninverted = math.MaxUint64 + +// PersistSegmentBase persists SegmentBase in the zap file format. +func PersistSegmentBase(sb *SegmentBase, path string) error { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + br := bufio.NewWriter(f) + + _, err = br.Write(sb.mem) + if err != nil { + cleanup() + return err + } + + err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.docValueOffset, + sb.chunkFactor, sb.memCRC, br) + if err != nil { + cleanup() + return err + } + + err = br.Flush() + if err != nil { + cleanup() + return err + } + + err = f.Sync() + if err != nil { + cleanup() + return err + } + + err = f.Close() + if err != nil { + cleanup() + return err + } + + return nil +} + +func persistStoredFieldValues(fieldID int, + storedFieldValues [][]byte, stf []byte, spf [][]uint64, + curr int, metaEncode varintEncoder, data []byte) ( + int, []byte, error) { + for i := 0; i < len(storedFieldValues); i++ { + // encode field + _, err := metaEncode(uint64(fieldID)) + if err != nil { + return 0, nil, err + } + // encode type + _, err = metaEncode(uint64(stf[i])) + if err != nil { + return 0, nil, err + } + // encode start offset + _, err = metaEncode(uint64(curr)) + if err != nil { + return 0, nil, err + } + // end len + _, err = metaEncode(uint64(len(storedFieldValues[i]))) + if err != nil { + return 0, nil, err + } + // encode number of array pos + _, err = metaEncode(uint64(len(spf[i]))) + if err != nil { + return 0, nil, err + } + // encode all array positions + for _, pos := range spf[i] { + _, err = metaEncode(pos) + if err != nil { + return 0, nil, err + } + } + + data = append(data, storedFieldValues[i]...) + curr += len(storedFieldValues[i]) + } + + return curr, data, nil +} + +func InitSegmentBase(mem []byte, memCRC uint32, chunkFactor uint32, + fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64, + storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64, + dictLocs []uint64) (*SegmentBase, error) { + sb := &SegmentBase{ + mem: mem, + memCRC: memCRC, + chunkFactor: chunkFactor, + fieldsMap: fieldsMap, + fieldsInv: fieldsInv, + numDocs: numDocs, + storedIndexOffset: storedIndexOffset, + fieldsIndexOffset: fieldsIndexOffset, + docValueOffset: docValueOffset, + dictLocs: dictLocs, + fieldDvReaders: make(map[uint16]*docValueReader), + } + sb.updateSize() + + err := sb.loadDvReaders() + if err != nil { + return nil, err + } + + return sb, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/contentcoder.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/contentcoder.go new file mode 100644 index 00000000..b9ff8179 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/contentcoder.go @@ -0,0 +1,230 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "io" + "reflect" + + "github.com/golang/snappy" +) + +var reflectStaticSizeMetaData int + +func init() { + var md MetaData + reflectStaticSizeMetaData = int(reflect.TypeOf(md).Size()) +} + +var termSeparator byte = 0xff +var termSeparatorSplitSlice = []byte{termSeparator} + +type chunkedContentCoder struct { + final []byte + chunkSize uint64 + currChunk uint64 + chunkLens []uint64 + + w io.Writer + progressiveWrite bool + + chunkMetaBuf bytes.Buffer + chunkBuf bytes.Buffer + + chunkMeta []MetaData + + compressed []byte // temp buf for snappy compression +} + +// MetaData represents the data information inside a +// chunk. +type MetaData struct { + DocNum uint64 // docNum of the data inside the chunk + DocDvOffset uint64 // offset of data inside the chunk for the given docid +} + +// newChunkedContentCoder returns a new chunk content coder which +// packs data into chunks based on the provided chunkSize +func newChunkedContentCoder(chunkSize uint64, maxDocNum uint64, + w io.Writer, progressiveWrite bool) *chunkedContentCoder { + total := maxDocNum/chunkSize + 1 + rv := &chunkedContentCoder{ + chunkSize: chunkSize, + chunkLens: make([]uint64, total), + chunkMeta: make([]MetaData, 0, total), + w: w, + progressiveWrite: progressiveWrite, + } + + return rv +} + +// Reset lets you reuse this chunked content coder. Buffers are reset +// and re used. You cannot change the chunk size. +func (c *chunkedContentCoder) Reset() { + c.currChunk = 0 + c.final = c.final[:0] + c.chunkBuf.Reset() + c.chunkMetaBuf.Reset() + for i := range c.chunkLens { + c.chunkLens[i] = 0 + } + c.chunkMeta = c.chunkMeta[:0] +} + +// Close indicates you are done calling Add() this allows +// the final chunk to be encoded. +func (c *chunkedContentCoder) Close() error { + return c.flushContents() +} + +func (c *chunkedContentCoder) flushContents() error { + // flush the contents, with meta information at first + buf := make([]byte, binary.MaxVarintLen64) + n := binary.PutUvarint(buf, uint64(len(c.chunkMeta))) + _, err := c.chunkMetaBuf.Write(buf[:n]) + if err != nil { + return err + } + + // write out the metaData slice + for _, meta := range c.chunkMeta { + _, err := writeUvarints(&c.chunkMetaBuf, meta.DocNum, meta.DocDvOffset) + if err != nil { + return err + } + } + + // write the metadata to final data + metaData := c.chunkMetaBuf.Bytes() + c.final = append(c.final, c.chunkMetaBuf.Bytes()...) + // write the compressed data to the final data + c.compressed = snappy.Encode(c.compressed[:cap(c.compressed)], c.chunkBuf.Bytes()) + c.final = append(c.final, c.compressed...) + + c.chunkLens[c.currChunk] = uint64(len(c.compressed) + len(metaData)) + + if c.progressiveWrite { + _, err := c.w.Write(c.final) + if err != nil { + return err + } + c.final = c.final[:0] + } + + return nil +} + +// Add encodes the provided byte slice into the correct chunk for the provided +// doc num. You MUST call Add() with increasing docNums. +func (c *chunkedContentCoder) Add(docNum uint64, vals []byte) error { + chunk := docNum / c.chunkSize + if chunk != c.currChunk { + // flush out the previous chunk details + err := c.flushContents() + if err != nil { + return err + } + // clearing the chunk specific meta for next chunk + c.chunkBuf.Reset() + c.chunkMetaBuf.Reset() + c.chunkMeta = c.chunkMeta[:0] + c.currChunk = chunk + } + + // get the starting offset for this doc + dvOffset := c.chunkBuf.Len() + dvSize, err := c.chunkBuf.Write(vals) + if err != nil { + return err + } + + c.chunkMeta = append(c.chunkMeta, MetaData{ + DocNum: docNum, + DocDvOffset: uint64(dvOffset + dvSize), + }) + return nil +} + +// Write commits all the encoded chunked contents to the provided writer. +// +// | ..... data ..... | chunk offsets (varints) +// | position of chunk offsets (uint64) | number of offsets (uint64) | +// +func (c *chunkedContentCoder) Write() (int, error) { + var tw int + + if c.final != nil { + // write out the data section first + nw, err := c.w.Write(c.final) + tw += nw + if err != nil { + return tw, err + } + } + + chunkOffsetsStart := uint64(tw) + + if cap(c.final) < binary.MaxVarintLen64 { + c.final = make([]byte, binary.MaxVarintLen64) + } else { + c.final = c.final[0:binary.MaxVarintLen64] + } + chunkOffsets := modifyLengthsToEndOffsets(c.chunkLens) + // write out the chunk offsets + for _, chunkOffset := range chunkOffsets { + n := binary.PutUvarint(c.final, chunkOffset) + nw, err := c.w.Write(c.final[:n]) + tw += nw + if err != nil { + return tw, err + } + } + + chunkOffsetsLen := uint64(tw) - chunkOffsetsStart + + c.final = c.final[0:8] + // write out the length of chunk offsets + binary.BigEndian.PutUint64(c.final, chunkOffsetsLen) + nw, err := c.w.Write(c.final) + tw += nw + if err != nil { + return tw, err + } + + // write out the number of chunks + binary.BigEndian.PutUint64(c.final, uint64(len(c.chunkLens))) + nw, err = c.w.Write(c.final) + tw += nw + if err != nil { + return tw, err + } + + c.final = c.final[:0] + + return tw, nil +} + +// ReadDocValueBoundary elicits the start, end offsets from a +// metaData header slice +func ReadDocValueBoundary(chunk int, metaHeaders []MetaData) (uint64, uint64) { + var start uint64 + if chunk > 0 { + start = metaHeaders[chunk-1].DocDvOffset + } + return start, metaHeaders[chunk].DocDvOffset +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/count.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/count.go new file mode 100644 index 00000000..d75e83c0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/count.go @@ -0,0 +1,51 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "hash/crc32" + "io" +) + +// CountHashWriter is a wrapper around a Writer which counts the number of +// bytes which have been written and computes a crc32 hash +type CountHashWriter struct { + w io.Writer + crc uint32 + n int +} + +// NewCountHashWriter returns a CountHashWriter which wraps the provided Writer +func NewCountHashWriter(w io.Writer) *CountHashWriter { + return &CountHashWriter{w: w} +} + +// Write writes the provided bytes to the wrapped writer and counts the bytes +func (c *CountHashWriter) Write(b []byte) (int, error) { + n, err := c.w.Write(b) + c.crc = crc32.Update(c.crc, crc32.IEEETable, b[:n]) + c.n += n + return n, err +} + +// Count returns the number of bytes written +func (c *CountHashWriter) Count() int { + return c.n +} + +// Sum32 returns the CRC-32 hash of the content written to this writer +func (c *CountHashWriter) Sum32() uint32 { + return c.crc +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/dict.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/dict.go new file mode 100644 index 00000000..736fa59f --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/dict.go @@ -0,0 +1,289 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "fmt" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/couchbase/vellum" + "github.com/couchbase/vellum/levenshtein" + "github.com/couchbase/vellum/regexp" +) + +// Dictionary is the zap representation of the term dictionary +type Dictionary struct { + sb *SegmentBase + field string + fieldID uint16 + fst *vellum.FST + fstReader *vellum.Reader +} + +// PostingsList returns the postings list for the specified term +func (d *Dictionary) PostingsList(term []byte, except *roaring.Bitmap, + prealloc segment.PostingsList) (segment.PostingsList, error) { + var preallocPL *PostingsList + pl, ok := prealloc.(*PostingsList) + if ok && pl != nil { + preallocPL = pl + } + return d.postingsList(term, except, preallocPL) +} + +func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + if d.fstReader == nil { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + postingsOffset, exists, err := d.fstReader.Get(term) + if err != nil { + return nil, fmt.Errorf("vellum err: %v", err) + } + if !exists { + if rv == nil || rv == emptyPostingsList { + return emptyPostingsList, nil + } + return d.postingsListInit(rv, except), nil + } + + return d.postingsListFromOffset(postingsOffset, except, rv) +} + +func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) { + rv = d.postingsListInit(rv, except) + + err := rv.read(postingsOffset, d) + if err != nil { + return nil, err + } + + return rv, nil +} + +func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList { + if rv == nil || rv == emptyPostingsList { + rv = &PostingsList{} + } else { + postings := rv.postings + if postings != nil { + postings.Clear() + } + + *rv = PostingsList{} // clear the struct + + rv.postings = postings + } + rv.sb = d.sb + rv.except = except + return rv +} + +// Iterator returns an iterator for this dictionary +func (d *Dictionary) Iterator() segment.DictionaryIterator { + rv := &DictionaryIterator{ + d: d, + } + + if d.fst != nil { + itr, err := d.fst.Iterator(nil, nil) + if err == nil { + rv.itr = itr + } else if err != nil && err != vellum.ErrIteratorDone { + rv.err = err + } + } + + return rv +} + +// PrefixIterator returns an iterator which only visits terms having the +// the specified prefix +func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator { + rv := &DictionaryIterator{ + d: d, + } + + if d.fst != nil { + r, err := regexp.New(prefix + ".*") + if err == nil { + itr, err := d.fst.Search(r, nil, nil) + if err == nil { + rv.itr = itr + } else if err != nil && err != vellum.ErrIteratorDone { + rv.err = err + } + } else { + rv.err = err + } + } + + return rv +} + +// RangeIterator returns an iterator which only visits terms between the +// start and end terms. NOTE: bleve.index API specifies the end is inclusive. +func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator { + rv := &DictionaryIterator{ + d: d, + } + + // need to increment the end position to be inclusive + endBytes := []byte(end) + if endBytes[len(endBytes)-1] < 0xff { + endBytes[len(endBytes)-1]++ + } else { + endBytes = append(endBytes, 0xff) + } + + if d.fst != nil { + itr, err := d.fst.Iterator([]byte(start), endBytes) + if err == nil { + rv.itr = itr + } else if err != nil && err != vellum.ErrIteratorDone { + rv.err = err + } + } + + return rv +} + +// RegexpIterator returns an iterator which only visits terms having the +// the specified regex +func (d *Dictionary) RegexpIterator(regex string) segment.DictionaryIterator { + rv := &DictionaryIterator{ + d: d, + } + + if d.fst != nil { + r, err := regexp.New(regex) + if err == nil { + itr, err2 := d.fst.Search(r, nil, nil) + if err2 == nil { + rv.itr = itr + } else if err2 != nil && err2 != vellum.ErrIteratorDone { + rv.err = err2 + } + } else { + rv.err = err + } + } + + return rv +} + +// FuzzyIterator returns an iterator which only visits terms having the +// the specified edit/levenshtein distance +func (d *Dictionary) FuzzyIterator(term string, + fuzziness int) segment.DictionaryIterator { + rv := &DictionaryIterator{ + d: d, + } + + if d.fst != nil { + la, err := levenshtein.New(term, fuzziness) + if err == nil { + itr, err2 := d.fst.Search(la, nil, nil) + if err2 == nil { + rv.itr = itr + } else if err2 != nil && err2 != vellum.ErrIteratorDone { + rv.err = err2 + } + } else { + rv.err = err + } + } + + return rv +} + +func (d *Dictionary) OnlyIterator(onlyTerms [][]byte, + includeCount bool) segment.DictionaryIterator { + + rv := &DictionaryIterator{ + d: d, + omitCount: !includeCount, + } + + var buf bytes.Buffer + builder, err := vellum.New(&buf, nil) + if err != nil { + rv.err = err + return rv + } + for _, term := range onlyTerms { + err = builder.Insert(term, 0) + if err != nil { + rv.err = err + return rv + } + } + err = builder.Close() + if err != nil { + rv.err = err + return rv + } + + onlyFST, err := vellum.Load(buf.Bytes()) + if err != nil { + rv.err = err + return rv + } + + itr, err := d.fst.Search(onlyFST, nil, nil) + if err == nil { + rv.itr = itr + } else if err != nil && err != vellum.ErrIteratorDone { + rv.err = err + } + + return rv +} + +// DictionaryIterator is an iterator for term dictionary +type DictionaryIterator struct { + d *Dictionary + itr vellum.Iterator + err error + tmp PostingsList + entry index.DictEntry + omitCount bool +} + +// Next returns the next entry in the dictionary +func (i *DictionaryIterator) Next() (*index.DictEntry, error) { + if i.err != nil && i.err != vellum.ErrIteratorDone { + return nil, i.err + } else if i.itr == nil || i.err == vellum.ErrIteratorDone { + return nil, nil + } + term, postingsOffset := i.itr.Current() + i.entry.Term = string(term) + if !i.omitCount { + i.err = i.tmp.read(postingsOffset, i.d) + if i.err != nil { + return nil, i.err + } + i.entry.Count = i.tmp.Count() + } + i.err = i.itr.Next() + return &i.entry, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/docvalues.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/docvalues.go new file mode 100644 index 00000000..d2896497 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/docvalues.go @@ -0,0 +1,312 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "reflect" + "sort" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/size" + "github.com/golang/snappy" +) + +var reflectStaticSizedocValueReader int + +func init() { + var dvi docValueReader + reflectStaticSizedocValueReader = int(reflect.TypeOf(dvi).Size()) +} + +type docNumTermsVisitor func(docNum uint64, terms []byte) error + +type docVisitState struct { + dvrs map[uint16]*docValueReader + segment *Segment +} + +type docValueReader struct { + field string + curChunkNum uint64 + chunkOffsets []uint64 + dvDataLoc uint64 + curChunkHeader []MetaData + curChunkData []byte // compressed data cache + uncompressed []byte // temp buf for snappy decompression +} + +func (di *docValueReader) size() int { + return reflectStaticSizedocValueReader + size.SizeOfPtr + + len(di.field) + + len(di.chunkOffsets)*size.SizeOfUint64 + + len(di.curChunkHeader)*reflectStaticSizeMetaData + + len(di.curChunkData) +} + +func (di *docValueReader) cloneInto(rv *docValueReader) *docValueReader { + if rv == nil { + rv = &docValueReader{} + } + + rv.field = di.field + rv.curChunkNum = math.MaxUint64 + rv.chunkOffsets = di.chunkOffsets // immutable, so it's sharable + rv.dvDataLoc = di.dvDataLoc + rv.curChunkHeader = nil + rv.curChunkData = nil + rv.uncompressed = nil + + return rv +} + +func (di *docValueReader) fieldName() string { + return di.field +} + +func (di *docValueReader) curChunkNumber() uint64 { + return di.curChunkNum +} + +func (s *SegmentBase) loadFieldDocValueReader(field string, + fieldDvLocStart, fieldDvLocEnd uint64) (*docValueReader, error) { + // get the docValue offset for the given fields + if fieldDvLocStart == fieldNotUninverted { + return nil, fmt.Errorf("loadFieldDocValueReader: "+ + "no docValues found for field: %s", field) + } + + // read the number of chunks, and chunk offsets position + var numChunks, chunkOffsetsPosition uint64 + + if fieldDvLocEnd-fieldDvLocStart > 16 { + numChunks = binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-8 : fieldDvLocEnd]) + // read the length of chunk offsets + chunkOffsetsLen := binary.BigEndian.Uint64(s.mem[fieldDvLocEnd-16 : fieldDvLocEnd-8]) + // acquire position of chunk offsets + chunkOffsetsPosition = (fieldDvLocEnd - 16) - chunkOffsetsLen + } + + fdvIter := &docValueReader{ + curChunkNum: math.MaxUint64, + field: field, + chunkOffsets: make([]uint64, int(numChunks)), + } + + // read the chunk offsets + var offset uint64 + for i := 0; i < int(numChunks); i++ { + loc, read := binary.Uvarint(s.mem[chunkOffsetsPosition+offset : chunkOffsetsPosition+offset+binary.MaxVarintLen64]) + if read <= 0 { + return nil, fmt.Errorf("corrupted chunk offset during segment load") + } + fdvIter.chunkOffsets[i] = loc + offset += uint64(read) + } + + // set the data offset + fdvIter.dvDataLoc = fieldDvLocStart + + return fdvIter, nil +} + +func (di *docValueReader) loadDvChunk(chunkNumber uint64, s *SegmentBase) error { + // advance to the chunk where the docValues + // reside for the given docNum + destChunkDataLoc, curChunkEnd := di.dvDataLoc, di.dvDataLoc + start, end := readChunkBoundary(int(chunkNumber), di.chunkOffsets) + if start >= end { + di.curChunkHeader = di.curChunkHeader[:0] + di.curChunkData = nil + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil + } + + destChunkDataLoc += start + curChunkEnd += end + + // read the number of docs reside in the chunk + numDocs, read := binary.Uvarint(s.mem[destChunkDataLoc : destChunkDataLoc+binary.MaxVarintLen64]) + if read <= 0 { + return fmt.Errorf("failed to read the chunk") + } + chunkMetaLoc := destChunkDataLoc + uint64(read) + + offset := uint64(0) + di.curChunkHeader = make([]MetaData, int(numDocs)) + for i := 0; i < int(numDocs); i++ { + di.curChunkHeader[i].DocNum, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + di.curChunkHeader[i].DocDvOffset, read = binary.Uvarint(s.mem[chunkMetaLoc+offset : chunkMetaLoc+offset+binary.MaxVarintLen64]) + offset += uint64(read) + } + + compressedDataLoc := chunkMetaLoc + offset + dataLength := curChunkEnd - compressedDataLoc + di.curChunkData = s.mem[compressedDataLoc : compressedDataLoc+dataLength] + di.curChunkNum = chunkNumber + di.uncompressed = di.uncompressed[:0] + return nil +} + +func (di *docValueReader) iterateAllDocValues(s *SegmentBase, visitor docNumTermsVisitor) error { + for i := 0; i < len(di.chunkOffsets); i++ { + err := di.loadDvChunk(uint64(i), s) + if err != nil { + return err + } + if di.curChunkData == nil || len(di.curChunkHeader) <= 0 { + continue + } + + // uncompress the already loaded data + uncompressed, err := snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + + start := uint64(0) + for _, entry := range di.curChunkHeader { + err = visitor(entry.DocNum, uncompressed[start:entry.DocDvOffset]) + if err != nil { + return err + } + + start = entry.DocDvOffset + } + } + + return nil +} + +func (di *docValueReader) visitDocValues(docNum uint64, + visitor index.DocumentFieldTermVisitor) error { + // binary search the term locations for the docNum + start, end := di.getDocValueLocs(docNum) + if start == math.MaxUint64 || end == math.MaxUint64 || start == end { + return nil + } + + var uncompressed []byte + var err error + // use the uncompressed copy if available + if len(di.uncompressed) > 0 { + uncompressed = di.uncompressed + } else { + // uncompress the already loaded data + uncompressed, err = snappy.Decode(di.uncompressed[:cap(di.uncompressed)], di.curChunkData) + if err != nil { + return err + } + di.uncompressed = uncompressed + } + + // pick the terms for the given docNum + uncompressed = uncompressed[start:end] + for { + i := bytes.Index(uncompressed, termSeparatorSplitSlice) + if i < 0 { + break + } + + visitor(di.field, uncompressed[0:i]) + uncompressed = uncompressed[i+1:] + } + + return nil +} + +func (di *docValueReader) getDocValueLocs(docNum uint64) (uint64, uint64) { + i := sort.Search(len(di.curChunkHeader), func(i int) bool { + return di.curChunkHeader[i].DocNum >= docNum + }) + if i < len(di.curChunkHeader) && di.curChunkHeader[i].DocNum == docNum { + return ReadDocValueBoundary(i, di.curChunkHeader) + } + return math.MaxUint64, math.MaxUint64 +} + +// VisitDocumentFieldTerms is an implementation of the +// DocumentFieldTermVisitable interface +func (s *Segment) VisitDocumentFieldTerms(localDocNum uint64, fields []string, + visitor index.DocumentFieldTermVisitor, dvsIn segment.DocVisitState) ( + segment.DocVisitState, error) { + dvs, ok := dvsIn.(*docVisitState) + if !ok || dvs == nil { + dvs = &docVisitState{} + } else { + if dvs.segment != s { + dvs.segment = s + dvs.dvrs = nil + } + } + + var fieldIDPlus1 uint16 + if dvs.dvrs == nil { + dvs.dvrs = make(map[uint16]*docValueReader, len(fields)) + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvIter, exists := s.fieldDvReaders[fieldID]; exists && + dvIter != nil { + dvs.dvrs[fieldID] = dvIter.cloneInto(dvs.dvrs[fieldID]) + } + } + } + + // find the chunkNumber where the docValues are stored + docInChunk := localDocNum / uint64(s.chunkFactor) + var dvr *docValueReader + for _, field := range fields { + if fieldIDPlus1, ok = s.fieldsMap[field]; !ok { + continue + } + fieldID := fieldIDPlus1 - 1 + if dvr, ok = dvs.dvrs[fieldID]; ok && dvr != nil { + // check if the chunk is already loaded + if docInChunk != dvr.curChunkNumber() { + err := dvr.loadDvChunk(docInChunk, &s.SegmentBase) + if err != nil { + return dvs, err + } + } + + _ = dvr.visitDocValues(localDocNum, visitor) + } + } + return dvs, nil +} + +// VisitableDocValueFields returns the list of fields with +// persisted doc value terms ready to be visitable using the +// VisitDocumentFieldTerms method. +func (s *Segment) VisitableDocValueFields() ([]string, error) { + rv := make([]string, 0, len(s.fieldDvReaders)) + for fieldID, field := range s.fieldsInv { + if dvIter, ok := s.fieldDvReaders[uint16(fieldID)]; ok && + dvIter != nil { + rv = append(rv, field) + } + } + return rv, nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/enumerator.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/enumerator.go new file mode 100644 index 00000000..3c708dd5 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/enumerator.go @@ -0,0 +1,124 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + + "github.com/couchbase/vellum" +) + +// enumerator provides an ordered traversal of multiple vellum +// iterators. Like JOIN of iterators, the enumerator produces a +// sequence of (key, iteratorIndex, value) tuples, sorted by key ASC, +// then iteratorIndex ASC, where the same key might be seen or +// repeated across multiple child iterators. +type enumerator struct { + itrs []vellum.Iterator + currKs [][]byte + currVs []uint64 + + lowK []byte + lowIdxs []int + lowCurr int +} + +// newEnumerator returns a new enumerator over the vellum Iterators +func newEnumerator(itrs []vellum.Iterator) (*enumerator, error) { + rv := &enumerator{ + itrs: itrs, + currKs: make([][]byte, len(itrs)), + currVs: make([]uint64, len(itrs)), + lowIdxs: make([]int, 0, len(itrs)), + } + for i, itr := range rv.itrs { + rv.currKs[i], rv.currVs[i] = itr.Current() + } + rv.updateMatches() + if rv.lowK == nil { + return rv, vellum.ErrIteratorDone + } + return rv, nil +} + +// updateMatches maintains the low key matches based on the currKs +func (m *enumerator) updateMatches() { + m.lowK = nil + m.lowIdxs = m.lowIdxs[:0] + m.lowCurr = 0 + + for i, key := range m.currKs { + if key == nil { + continue + } + + cmp := bytes.Compare(key, m.lowK) + if cmp < 0 || m.lowK == nil { + // reached a new low + m.lowK = key + m.lowIdxs = m.lowIdxs[:0] + m.lowIdxs = append(m.lowIdxs, i) + } else if cmp == 0 { + m.lowIdxs = append(m.lowIdxs, i) + } + } +} + +// Current returns the enumerator's current key, iterator-index, and +// value. If the enumerator is not pointing at a valid value (because +// Next returned an error previously), Current will return nil,0,0. +func (m *enumerator) Current() ([]byte, int, uint64) { + var i int + var v uint64 + if m.lowCurr < len(m.lowIdxs) { + i = m.lowIdxs[m.lowCurr] + v = m.currVs[i] + } + return m.lowK, i, v +} + +// Next advances the enumerator to the next key/iterator/value result, +// else vellum.ErrIteratorDone is returned. +func (m *enumerator) Next() error { + m.lowCurr += 1 + if m.lowCurr >= len(m.lowIdxs) { + // move all the current low iterators forwards + for _, vi := range m.lowIdxs { + err := m.itrs[vi].Next() + if err != nil && err != vellum.ErrIteratorDone { + return err + } + m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() + } + m.updateMatches() + } + if m.lowK == nil { + return vellum.ErrIteratorDone + } + return nil +} + +// Close all the underlying Iterators. The first error, if any, will +// be returned. +func (m *enumerator) Close() error { + var rv error + for _, itr := range m.itrs { + err := itr.Close() + if rv == nil { + rv = err + } + } + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/intcoder.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/intcoder.go new file mode 100644 index 00000000..571d06ed --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/intcoder.go @@ -0,0 +1,172 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "io" +) + +type chunkedIntCoder struct { + final []byte + chunkSize uint64 + chunkBuf bytes.Buffer + chunkLens []uint64 + currChunk uint64 + + buf []byte +} + +// newChunkedIntCoder returns a new chunk int coder which packs data into +// chunks based on the provided chunkSize and supports up to the specified +// maxDocNum +func newChunkedIntCoder(chunkSize uint64, maxDocNum uint64) *chunkedIntCoder { + total := maxDocNum/chunkSize + 1 + rv := &chunkedIntCoder{ + chunkSize: chunkSize, + chunkLens: make([]uint64, total), + final: make([]byte, 0, 64), + } + + return rv +} + +// Reset lets you reuse this chunked int coder. buffers are reset and reused +// from previous use. you cannot change the chunk size or max doc num. +func (c *chunkedIntCoder) Reset() { + c.final = c.final[:0] + c.chunkBuf.Reset() + c.currChunk = 0 + for i := range c.chunkLens { + c.chunkLens[i] = 0 + } +} + +// Add encodes the provided integers into the correct chunk for the provided +// doc num. You MUST call Add() with increasing docNums. +func (c *chunkedIntCoder) Add(docNum uint64, vals ...uint64) error { + chunk := docNum / c.chunkSize + if chunk != c.currChunk { + // starting a new chunk + c.Close() + c.chunkBuf.Reset() + c.currChunk = chunk + } + + if len(c.buf) < binary.MaxVarintLen64 { + c.buf = make([]byte, binary.MaxVarintLen64) + } + + for _, val := range vals { + wb := binary.PutUvarint(c.buf, val) + _, err := c.chunkBuf.Write(c.buf[:wb]) + if err != nil { + return err + } + } + + return nil +} + +func (c *chunkedIntCoder) AddBytes(docNum uint64, buf []byte) error { + chunk := docNum / c.chunkSize + if chunk != c.currChunk { + // starting a new chunk + c.Close() + c.chunkBuf.Reset() + c.currChunk = chunk + } + + _, err := c.chunkBuf.Write(buf) + return err +} + +// Close indicates you are done calling Add() this allows the final chunk +// to be encoded. +func (c *chunkedIntCoder) Close() { + encodingBytes := c.chunkBuf.Bytes() + c.chunkLens[c.currChunk] = uint64(len(encodingBytes)) + c.final = append(c.final, encodingBytes...) + c.currChunk = uint64(cap(c.chunkLens)) // sentinel to detect double close +} + +// Write commits all the encoded chunked integers to the provided writer. +func (c *chunkedIntCoder) Write(w io.Writer) (int, error) { + bufNeeded := binary.MaxVarintLen64 * (1 + len(c.chunkLens)) + if len(c.buf) < bufNeeded { + c.buf = make([]byte, bufNeeded) + } + buf := c.buf + + // convert the chunk lengths into chunk offsets + chunkOffsets := modifyLengthsToEndOffsets(c.chunkLens) + + // write out the number of chunks & each chunk offsets + n := binary.PutUvarint(buf, uint64(len(chunkOffsets))) + for _, chunkOffset := range chunkOffsets { + n += binary.PutUvarint(buf[n:], chunkOffset) + } + + tw, err := w.Write(buf[:n]) + if err != nil { + return tw, err + } + + // write out the data + nw, err := w.Write(c.final) + tw += nw + if err != nil { + return tw, err + } + return tw, nil +} + +func (c *chunkedIntCoder) FinalSize() int { + return len(c.final) +} + +// modifyLengthsToEndOffsets converts the chunk length array +// to a chunk offset array. The readChunkBoundary +// will figure out the start and end of every chunk from +// these offsets. Starting offset of i'th index is stored +// in i-1'th position except for 0'th index and ending offset +// is stored at i'th index position. +// For 0'th element, starting position is always zero. +// eg: +// Lens -> 5 5 5 5 => 5 10 15 20 +// Lens -> 0 5 0 5 => 0 5 5 10 +// Lens -> 0 0 0 5 => 0 0 0 5 +// Lens -> 5 0 0 0 => 5 5 5 5 +// Lens -> 0 5 0 0 => 0 5 5 5 +// Lens -> 0 0 5 0 => 0 0 5 5 +func modifyLengthsToEndOffsets(lengths []uint64) []uint64 { + var runningOffset uint64 + var index, i int + for i = 1; i <= len(lengths); i++ { + runningOffset += lengths[i-1] + lengths[index] = runningOffset + index++ + } + return lengths +} + +func readChunkBoundary(chunk int, offsets []uint64) (uint64, uint64) { + var start uint64 + if chunk > 0 { + start = offsets[chunk-1] + } + return start, offsets[chunk] +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/merge.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/merge.go new file mode 100644 index 00000000..dab09f6b --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/merge.go @@ -0,0 +1,801 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "math" + "os" + "sort" + + "github.com/RoaringBitmap/roaring" + "github.com/couchbase/vellum" + "github.com/golang/snappy" +) + +var DefaultFileMergerBufferSize = 1024 * 1024 + +const docDropped = math.MaxUint64 // sentinel docNum to represent a deleted doc + +// Merge takes a slice of zap segments and bit masks describing which +// documents may be dropped, and creates a new segment containing the +// remaining data. This new segment is built at the specified path, +// with the provided chunkFactor. +func Merge(segments []*Segment, drops []*roaring.Bitmap, path string, + chunkFactor uint32) ([][]uint64, uint64, error) { + segmentBases := make([]*SegmentBase, len(segments)) + for segmenti, segment := range segments { + segmentBases[segmenti] = &segment.SegmentBase + } + + return MergeSegmentBases(segmentBases, drops, path, chunkFactor) +} + +func MergeSegmentBases(segmentBases []*SegmentBase, drops []*roaring.Bitmap, path string, + chunkFactor uint32) ([][]uint64, uint64, error) { + flag := os.O_RDWR | os.O_CREATE + + f, err := os.OpenFile(path, flag, 0600) + if err != nil { + return nil, 0, err + } + + cleanup := func() { + _ = f.Close() + _ = os.Remove(path) + } + + // buffer the output + br := bufio.NewWriterSize(f, DefaultFileMergerBufferSize) + + // wrap it for counting (tracking offsets) + cr := NewCountHashWriter(br) + + newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, _, _, _, err := + MergeToWriter(segmentBases, drops, chunkFactor, cr) + if err != nil { + cleanup() + return nil, 0, err + } + + err = persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, + docValueOffset, chunkFactor, cr.Sum32(), cr) + if err != nil { + cleanup() + return nil, 0, err + } + + err = br.Flush() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Sync() + if err != nil { + cleanup() + return nil, 0, err + } + + err = f.Close() + if err != nil { + cleanup() + return nil, 0, err + } + + return newDocNums, uint64(cr.Count()), nil +} + +func MergeToWriter(segments []*SegmentBase, drops []*roaring.Bitmap, + chunkFactor uint32, cr *CountHashWriter) ( + newDocNums [][]uint64, + numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, + dictLocs []uint64, fieldsInv []string, fieldsMap map[string]uint16, + err error) { + docValueOffset = uint64(fieldNotUninverted) + + var fieldsSame bool + fieldsSame, fieldsInv = mergeFields(segments) + fieldsMap = mapFields(fieldsInv) + + numDocs = computeNewDocCount(segments, drops) + if numDocs > 0 { + storedIndexOffset, newDocNums, err = mergeStoredAndRemap(segments, drops, + fieldsMap, fieldsInv, fieldsSame, numDocs, cr) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + dictLocs, docValueOffset, err = persistMergedRest(segments, drops, + fieldsInv, fieldsMap, fieldsSame, + newDocNums, numDocs, chunkFactor, cr) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + } else { + dictLocs = make([]uint64, len(fieldsInv)) + } + + fieldsIndexOffset, err = persistFields(fieldsInv, cr, dictLocs) + if err != nil { + return nil, 0, 0, 0, 0, nil, nil, nil, err + } + + return newDocNums, numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset, dictLocs, fieldsInv, fieldsMap, nil +} + +// mapFields takes the fieldsInv list and returns a map of fieldName +// to fieldID+1 +func mapFields(fields []string) map[string]uint16 { + rv := make(map[string]uint16, len(fields)) + for i, fieldName := range fields { + rv[fieldName] = uint16(i) + 1 + } + return rv +} + +// computeNewDocCount determines how many documents will be in the newly +// merged segment when obsoleted docs are dropped +func computeNewDocCount(segments []*SegmentBase, drops []*roaring.Bitmap) uint64 { + var newDocCount uint64 + for segI, segment := range segments { + newDocCount += segment.numDocs + if drops[segI] != nil { + newDocCount -= drops[segI].GetCardinality() + } + } + return newDocCount +} + +func persistMergedRest(segments []*SegmentBase, dropsIn []*roaring.Bitmap, + fieldsInv []string, fieldsMap map[string]uint16, fieldsSame bool, + newDocNumsIn [][]uint64, newSegDocCount uint64, chunkFactor uint32, + w *CountHashWriter) ([]uint64, uint64, error) { + + var bufMaxVarintLen64 []byte = make([]byte, binary.MaxVarintLen64) + var bufLoc []uint64 + + var postings *PostingsList + var postItr *PostingsIterator + + rv := make([]uint64, len(fieldsInv)) + fieldDvLocsStart := make([]uint64, len(fieldsInv)) + fieldDvLocsEnd := make([]uint64, len(fieldsInv)) + + tfEncoder := newChunkedIntCoder(uint64(chunkFactor), newSegDocCount-1) + locEncoder := newChunkedIntCoder(uint64(chunkFactor), newSegDocCount-1) + + var vellumBuf bytes.Buffer + newVellum, err := vellum.New(&vellumBuf, nil) + if err != nil { + return nil, 0, err + } + + newRoaring := roaring.NewBitmap() + + // for each field + for fieldID, fieldName := range fieldsInv { + + // collect FST iterators from all active segments for this field + var newDocNums [][]uint64 + var drops []*roaring.Bitmap + var dicts []*Dictionary + var itrs []vellum.Iterator + + var segmentsInFocus []*SegmentBase + + for segmentI, segment := range segments { + dict, err2 := segment.dictionary(fieldName) + if err2 != nil { + return nil, 0, err2 + } + if dict != nil && dict.fst != nil { + itr, err2 := dict.fst.Iterator(nil, nil) + if err2 != nil && err2 != vellum.ErrIteratorDone { + return nil, 0, err2 + } + if itr != nil { + newDocNums = append(newDocNums, newDocNumsIn[segmentI]) + if dropsIn[segmentI] != nil && !dropsIn[segmentI].IsEmpty() { + drops = append(drops, dropsIn[segmentI]) + } else { + drops = append(drops, nil) + } + dicts = append(dicts, dict) + itrs = append(itrs, itr) + segmentsInFocus = append(segmentsInFocus, segment) + } + } + } + + var prevTerm []byte + + newRoaring.Clear() + + var lastDocNum, lastFreq, lastNorm uint64 + + // determines whether to use "1-hit" encoding optimization + // when a term appears in only 1 doc, with no loc info, + // has freq of 1, and the docNum fits into 31-bits + use1HitEncoding := func(termCardinality uint64) (bool, uint64, uint64) { + if termCardinality == uint64(1) && locEncoder.FinalSize() <= 0 { + docNum := uint64(newRoaring.Minimum()) + if under32Bits(docNum) && docNum == lastDocNum && lastFreq == 1 { + return true, docNum, lastNorm + } + } + return false, 0, 0 + } + + finishTerm := func(term []byte) error { + if term == nil { + return nil + } + + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := writePostings(newRoaring, + tfEncoder, locEncoder, use1HitEncoding, w, bufMaxVarintLen64) + if err != nil { + return err + } + + if postingsOffset > 0 { + err = newVellum.Insert(term, postingsOffset) + if err != nil { + return err + } + } + + newRoaring.Clear() + + tfEncoder.Reset() + locEncoder.Reset() + + lastDocNum = 0 + lastFreq = 0 + lastNorm = 0 + + return nil + } + + enumerator, err := newEnumerator(itrs) + + for err == nil { + term, itrI, postingsOffset := enumerator.Current() + + if !bytes.Equal(prevTerm, term) { + // if the term changed, write out the info collected + // for the previous term + err2 := finishTerm(prevTerm) + if err2 != nil { + return nil, 0, err2 + } + } + + var err2 error + postings, err2 = dicts[itrI].postingsListFromOffset( + postingsOffset, drops[itrI], postings) + if err2 != nil { + return nil, 0, err2 + } + + postItr = postings.iterator(true, true, true, postItr) + + if fieldsSame { + // can optimize by copying freq/norm/loc bytes directly + lastDocNum, lastFreq, lastNorm, err = mergeTermFreqNormLocsByCopying( + term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder) + } else { + lastDocNum, lastFreq, lastNorm, bufLoc, err = mergeTermFreqNormLocs( + fieldsMap, term, postItr, newDocNums[itrI], newRoaring, + tfEncoder, locEncoder, bufLoc) + } + if err != nil { + return nil, 0, err + } + + prevTerm = prevTerm[:0] // copy to prevTerm in case Next() reuses term mem + prevTerm = append(prevTerm, term...) + + err = enumerator.Next() + } + if err != nil && err != vellum.ErrIteratorDone { + return nil, 0, err + } + + err = finishTerm(prevTerm) + if err != nil { + return nil, 0, err + } + + dictOffset := uint64(w.Count()) + + err = newVellum.Close() + if err != nil { + return nil, 0, err + } + vellumData := vellumBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(bufMaxVarintLen64, uint64(len(vellumData))) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return nil, 0, err + } + + // write this vellum to disk + _, err = w.Write(vellumData) + if err != nil { + return nil, 0, err + } + + rv[fieldID] = dictOffset + + // get the field doc value offset (start) + fieldDvLocsStart[fieldID] = uint64(w.Count()) + + // update the field doc values + fdvEncoder := newChunkedContentCoder(uint64(chunkFactor), newSegDocCount-1, w, true) + + fdvReadersAvailable := false + var dvIterClone *docValueReader + for segmentI, segment := range segmentsInFocus { + fieldIDPlus1 := uint16(segment.fieldsMap[fieldName]) + if dvIter, exists := segment.fieldDvReaders[fieldIDPlus1-1]; exists && + dvIter != nil { + fdvReadersAvailable = true + dvIterClone = dvIter.cloneInto(dvIterClone) + err = dvIterClone.iterateAllDocValues(segment, func(docNum uint64, terms []byte) error { + if newDocNums[segmentI][docNum] == docDropped { + return nil + } + err := fdvEncoder.Add(newDocNums[segmentI][docNum], terms) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, 0, err + } + } + } + + if fdvReadersAvailable { + err = fdvEncoder.Close() + if err != nil { + return nil, 0, err + } + + // persist the doc value details for this field + _, err = fdvEncoder.Write() + if err != nil { + return nil, 0, err + } + + // get the field doc value offset (end) + fieldDvLocsEnd[fieldID] = uint64(w.Count()) + } else { + fieldDvLocsStart[fieldID] = fieldNotUninverted + fieldDvLocsEnd[fieldID] = fieldNotUninverted + } + + // reset vellum buffer and vellum builder + vellumBuf.Reset() + err = newVellum.Reset(&vellumBuf) + if err != nil { + return nil, 0, err + } + } + + fieldDvLocsOffset := uint64(w.Count()) + + buf := bufMaxVarintLen64 + for i := 0; i < len(fieldDvLocsStart); i++ { + n := binary.PutUvarint(buf, fieldDvLocsStart[i]) + _, err := w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + n = binary.PutUvarint(buf, fieldDvLocsEnd[i]) + _, err = w.Write(buf[:n]) + if err != nil { + return nil, 0, err + } + } + + return rv, fieldDvLocsOffset, nil +} + +func mergeTermFreqNormLocs(fieldsMap map[string]uint16, term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder, bufLoc []uint64) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, bufLocOut []uint64, err error) { + next, err := postItr.Next() + for next != nil && err == nil { + hitNewDocNum := newDocNums[next.Number()] + if hitNewDocNum == docDropped { + return 0, 0, 0, nil, fmt.Errorf("see hit with dropped docNum") + } + + newRoaring.Add(uint32(hitNewDocNum)) + + nextFreq := next.Frequency() + nextNorm := uint64(math.Float32bits(float32(next.Norm()))) + + locs := next.Locations() + + err = tfEncoder.Add(hitNewDocNum, + encodeFreqHasLocs(nextFreq, len(locs) > 0), nextNorm) + if err != nil { + return 0, 0, 0, nil, err + } + + if len(locs) > 0 { + numBytesLocs := 0 + for _, loc := range locs { + ap := loc.ArrayPositions() + numBytesLocs += totalUvarintBytes(uint64(fieldsMap[loc.Field()]-1), + loc.Pos(), loc.Start(), loc.End(), uint64(len(ap)), ap) + } + + err = locEncoder.Add(hitNewDocNum, uint64(numBytesLocs)) + if err != nil { + return 0, 0, 0, nil, err + } + + for _, loc := range locs { + ap := loc.ArrayPositions() + if cap(bufLoc) < 5+len(ap) { + bufLoc = make([]uint64, 0, 5+len(ap)) + } + args := bufLoc[0:5] + args[0] = uint64(fieldsMap[loc.Field()] - 1) + args[1] = loc.Pos() + args[2] = loc.Start() + args[3] = loc.End() + args[4] = uint64(len(ap)) + args = append(args, ap...) + err = locEncoder.Add(hitNewDocNum, args...) + if err != nil { + return 0, 0, 0, nil, err + } + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + next, err = postItr.Next() + } + + return lastDocNum, lastFreq, lastNorm, bufLoc, err +} + +func mergeTermFreqNormLocsByCopying(term []byte, postItr *PostingsIterator, + newDocNums []uint64, newRoaring *roaring.Bitmap, + tfEncoder *chunkedIntCoder, locEncoder *chunkedIntCoder) ( + lastDocNum uint64, lastFreq uint64, lastNorm uint64, err error) { + nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err := + postItr.nextBytes() + for err == nil && len(nextFreqNormBytes) > 0 { + hitNewDocNum := newDocNums[nextDocNum] + if hitNewDocNum == docDropped { + return 0, 0, 0, fmt.Errorf("see hit with dropped doc num") + } + + newRoaring.Add(uint32(hitNewDocNum)) + err = tfEncoder.AddBytes(hitNewDocNum, nextFreqNormBytes) + if err != nil { + return 0, 0, 0, err + } + + if len(nextLocBytes) > 0 { + err = locEncoder.AddBytes(hitNewDocNum, nextLocBytes) + if err != nil { + return 0, 0, 0, err + } + } + + lastDocNum = hitNewDocNum + lastFreq = nextFreq + lastNorm = nextNorm + + nextDocNum, nextFreq, nextNorm, nextFreqNormBytes, nextLocBytes, err = + postItr.nextBytes() + } + + return lastDocNum, lastFreq, lastNorm, err +} + +func writePostings(postings *roaring.Bitmap, tfEncoder, locEncoder *chunkedIntCoder, + use1HitEncoding func(uint64) (bool, uint64, uint64), + w *CountHashWriter, bufMaxVarintLen64 []byte) ( + offset uint64, err error) { + termCardinality := postings.GetCardinality() + if termCardinality <= 0 { + return 0, nil + } + + if use1HitEncoding != nil { + encodeAs1Hit, docNum1Hit, normBits1Hit := use1HitEncoding(termCardinality) + if encodeAs1Hit { + return FSTValEncode1Hit(docNum1Hit, normBits1Hit), nil + } + } + + tfOffset := uint64(w.Count()) + _, err = tfEncoder.Write(w) + if err != nil { + return 0, err + } + + locOffset := uint64(w.Count()) + _, err = locEncoder.Write(w) + if err != nil { + return 0, err + } + + postingsOffset := uint64(w.Count()) + + n := binary.PutUvarint(bufMaxVarintLen64, tfOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + n = binary.PutUvarint(bufMaxVarintLen64, locOffset) + _, err = w.Write(bufMaxVarintLen64[:n]) + if err != nil { + return 0, err + } + + _, err = writeRoaringWithLen(postings, w, bufMaxVarintLen64) + if err != nil { + return 0, err + } + + return postingsOffset, nil +} + +type varintEncoder func(uint64) (int, error) + +func mergeStoredAndRemap(segments []*SegmentBase, drops []*roaring.Bitmap, + fieldsMap map[string]uint16, fieldsInv []string, fieldsSame bool, newSegDocCount uint64, + w *CountHashWriter) (uint64, [][]uint64, error) { + var rv [][]uint64 // The remapped or newDocNums for each segment. + + var newDocNum uint64 + + var curr int + var data, compressed []byte + var metaBuf bytes.Buffer + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return metaBuf.Write(varBuf[:wb]) + } + + vals := make([][][]byte, len(fieldsInv)) + typs := make([][]byte, len(fieldsInv)) + poss := make([][][]uint64, len(fieldsInv)) + + docNumOffsets := make([]uint64, newSegDocCount) + + // for each segment + for segI, segment := range segments { + segNewDocNums := make([]uint64, segment.numDocs) + + dropsI := drops[segI] + + // optimize when the field mapping is the same across all + // segments and there are no deletions, via byte-copying + // of stored docs bytes directly to the writer + if fieldsSame && (dropsI == nil || dropsI.GetCardinality() == 0) { + err := segment.copyStoredDocs(newDocNum, docNumOffsets, w) + if err != nil { + return 0, nil, err + } + + for i := uint64(0); i < segment.numDocs; i++ { + segNewDocNums[i] = newDocNum + newDocNum++ + } + rv = append(rv, segNewDocNums) + + continue + } + + // for each doc num + for docNum := uint64(0); docNum < segment.numDocs; docNum++ { + // TODO: roaring's API limits docNums to 32-bits? + if dropsI != nil && dropsI.Contains(uint32(docNum)) { + segNewDocNums[docNum] = docDropped + continue + } + + segNewDocNums[docNum] = newDocNum + + curr = 0 + metaBuf.Reset() + data = data[:0] + + // collect all the data + for i := 0; i < len(fieldsInv); i++ { + vals[i] = vals[i][:0] + typs[i] = typs[i][:0] + poss[i] = poss[i][:0] + } + err := segment.VisitDocument(docNum, func(field string, typ byte, value []byte, pos []uint64) bool { + fieldID := int(fieldsMap[field]) - 1 + vals[fieldID] = append(vals[fieldID], value) + typs[fieldID] = append(typs[fieldID], typ) + poss[fieldID] = append(poss[fieldID], pos) + return true + }) + if err != nil { + return 0, nil, err + } + + // _id field special case optimizes ExternalID() lookups + idFieldVal := vals[uint16(0)][0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, nil, err + } + + // now walk the non-"_id" fields in order + for fieldID := 1; fieldID < len(fieldsInv); fieldID++ { + storedFieldValues := vals[fieldID] + + stf := typs[fieldID] + spf := poss[fieldID] + + var err2 error + curr, data, err2 = persistStoredFieldValues(fieldID, + storedFieldValues, stf, spf, curr, metaEncode, data) + if err2 != nil { + return 0, nil, err2 + } + } + + metaBytes := metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + // record where we're about to start writing + docNumOffsets[newDocNum] = uint64(w.Count()) + + // write out the meta len and compressed data len + _, err = writeUvarints(w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, nil, err + } + // now write the meta + _, err = w.Write(metaBytes) + if err != nil { + return 0, nil, err + } + // now write the _id field val (counted as part of the 'compressed' data) + _, err = w.Write(idFieldVal) + if err != nil { + return 0, nil, err + } + // now write the compressed data + _, err = w.Write(compressed) + if err != nil { + return 0, nil, err + } + + newDocNum++ + } + + rv = append(rv, segNewDocNums) + } + + // return value is the start of the stored index + storedIndexOffset := uint64(w.Count()) + + // now write out the stored doc index + for _, docNumOffset := range docNumOffsets { + err := binary.Write(w, binary.BigEndian, docNumOffset) + if err != nil { + return 0, nil, err + } + } + + return storedIndexOffset, rv, nil +} + +// copyStoredDocs writes out a segment's stored doc info, optimized by +// using a single Write() call for the entire set of bytes. The +// newDocNumOffsets is filled with the new offsets for each doc. +func (s *SegmentBase) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, + w *CountHashWriter) error { + if s.numDocs <= 0 { + return nil + } + + indexOffset0, storedOffset0, _, _, _ := + s.getDocStoredOffsets(0) // the segment's first doc + + indexOffsetN, storedOffsetN, readN, metaLenN, dataLenN := + s.getDocStoredOffsets(s.numDocs - 1) // the segment's last doc + + storedOffset0New := uint64(w.Count()) + + storedBytes := s.mem[storedOffset0 : storedOffsetN+readN+metaLenN+dataLenN] + _, err := w.Write(storedBytes) + if err != nil { + return err + } + + // remap the storedOffset's for the docs into new offsets relative + // to storedOffset0New, filling the given docNumOffsetsOut array + for indexOffset := indexOffset0; indexOffset <= indexOffsetN; indexOffset += 8 { + storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) + storedOffsetNew := storedOffset - storedOffset0 + storedOffset0New + newDocNumOffsets[newDocNum] = storedOffsetNew + newDocNum += 1 + } + + return nil +} + +// mergeFields builds a unified list of fields used across all the +// input segments, and computes whether the fields are the same across +// segments (which depends on fields to be sorted in the same way +// across segments) +func mergeFields(segments []*SegmentBase) (bool, []string) { + fieldsSame := true + + var segment0Fields []string + if len(segments) > 0 { + segment0Fields = segments[0].Fields() + } + + fieldsExist := map[string]struct{}{} + for _, segment := range segments { + fields := segment.Fields() + for fieldi, field := range fields { + fieldsExist[field] = struct{}{} + if len(segment0Fields) != len(fields) || segment0Fields[fieldi] != field { + fieldsSame = false + } + } + } + + rv := make([]string, 0, len(fieldsExist)) + // ensure _id stays first + rv = append(rv, "_id") + for k := range fieldsExist { + if k != "_id" { + rv = append(rv, k) + } + } + + sort.Strings(rv[1:]) // leave _id as first + + return fieldsSame, rv +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/new.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/new.go new file mode 100644 index 00000000..22b69913 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/new.go @@ -0,0 +1,826 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "math" + "sort" + "sync" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/analysis" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" + "github.com/couchbase/vellum" + "github.com/golang/snappy" +) + +var NewSegmentBufferNumResultsBump int = 100 +var NewSegmentBufferNumResultsFactor float64 = 1.0 +var NewSegmentBufferAvgBytesPerDocFactor float64 = 1.0 + +// AnalysisResultsToSegmentBase produces an in-memory zap-encoded +// SegmentBase from analysis results +func AnalysisResultsToSegmentBase(results []*index.AnalysisResult, + chunkFactor uint32) (*SegmentBase, uint64, error) { + s := interimPool.Get().(*interim) + + var br bytes.Buffer + if s.lastNumDocs > 0 { + // use previous results to initialize the buf with an estimate + // size, but note that the interim instance comes from a + // global interimPool, so multiple scorch instances indexing + // different docs can lead to low quality estimates + estimateAvgBytesPerDoc := int(float64(s.lastOutSize/s.lastNumDocs) * + NewSegmentBufferNumResultsFactor) + estimateNumResults := int(float64(len(results)+NewSegmentBufferNumResultsBump) * + NewSegmentBufferAvgBytesPerDocFactor) + br.Grow(estimateAvgBytesPerDoc * estimateNumResults) + } + + s.results = results + s.chunkFactor = chunkFactor + s.w = NewCountHashWriter(&br) + + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, + err := s.convert() + if err != nil { + return nil, uint64(0), err + } + + sb, err := InitSegmentBase(br.Bytes(), s.w.Sum32(), chunkFactor, + s.FieldsMap, s.FieldsInv, uint64(len(results)), + storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets) + + if err == nil && s.reset() == nil { + s.lastNumDocs = len(results) + s.lastOutSize = len(br.Bytes()) + interimPool.Put(s) + } + + return sb, uint64(len(br.Bytes())), err +} + +var interimPool = sync.Pool{New: func() interface{} { return &interim{} }} + +// interim holds temporary working data used while converting from +// analysis results to a zap-encoded segment +type interim struct { + results []*index.AnalysisResult + + chunkFactor uint32 + + w *CountHashWriter + + // FieldsMap adds 1 to field id to avoid zero value issues + // name -> field id + 1 + FieldsMap map[string]uint16 + + // FieldsInv is the inverse of FieldsMap + // field id -> name + FieldsInv []string + + // Term dictionaries for each field + // field id -> term -> postings list id + 1 + Dicts []map[string]uint64 + + // Terms for each field, where terms are sorted ascending + // field id -> []term + DictKeys [][]string + + // Fields whose IncludeDocValues is true + // field id -> bool + IncludeDocValues []bool + + // postings id -> bitmap of docNums + Postings []*roaring.Bitmap + + // postings id -> freq/norm's, one for each docNum in postings + FreqNorms [][]interimFreqNorm + freqNormsBacking []interimFreqNorm + + // postings id -> locs, one for each freq + Locs [][]interimLoc + locsBacking []interimLoc + + numTermsPerPostingsList []int // key is postings list id + numLocsPerPostingsList []int // key is postings list id + + builder *vellum.Builder + builderBuf bytes.Buffer + + metaBuf bytes.Buffer + + tmp0 []byte + tmp1 []byte + + lastNumDocs int + lastOutSize int +} + +func (s *interim) reset() (err error) { + s.results = nil + s.chunkFactor = 0 + s.w = nil + s.FieldsMap = nil + s.FieldsInv = nil + for i := range s.Dicts { + s.Dicts[i] = nil + } + s.Dicts = s.Dicts[:0] + for i := range s.DictKeys { + s.DictKeys[i] = s.DictKeys[i][:0] + } + s.DictKeys = s.DictKeys[:0] + for i := range s.IncludeDocValues { + s.IncludeDocValues[i] = false + } + s.IncludeDocValues = s.IncludeDocValues[:0] + for _, idn := range s.Postings { + idn.Clear() + } + s.Postings = s.Postings[:0] + s.FreqNorms = s.FreqNorms[:0] + for i := range s.freqNormsBacking { + s.freqNormsBacking[i] = interimFreqNorm{} + } + s.freqNormsBacking = s.freqNormsBacking[:0] + s.Locs = s.Locs[:0] + for i := range s.locsBacking { + s.locsBacking[i] = interimLoc{} + } + s.locsBacking = s.locsBacking[:0] + s.numTermsPerPostingsList = s.numTermsPerPostingsList[:0] + s.numLocsPerPostingsList = s.numLocsPerPostingsList[:0] + s.builderBuf.Reset() + if s.builder != nil { + err = s.builder.Reset(&s.builderBuf) + } + s.metaBuf.Reset() + s.tmp0 = s.tmp0[:0] + s.tmp1 = s.tmp1[:0] + s.lastNumDocs = 0 + s.lastOutSize = 0 + + return err +} + +func (s *interim) grabBuf(size int) []byte { + buf := s.tmp0 + if cap(buf) < size { + buf = make([]byte, size) + s.tmp0 = buf + } + return buf[0:size] +} + +type interimStoredField struct { + vals [][]byte + typs []byte + arrayposs [][]uint64 // array positions +} + +type interimFreqNorm struct { + freq uint64 + norm float32 + numLocs int +} + +type interimLoc struct { + fieldID uint16 + pos uint64 + start uint64 + end uint64 + arrayposs []uint64 +} + +func (s *interim) convert() (uint64, uint64, uint64, []uint64, error) { + s.FieldsMap = map[string]uint16{} + + s.getOrDefineField("_id") // _id field is fieldID 0 + + for _, result := range s.results { + for _, field := range result.Document.CompositeFields { + s.getOrDefineField(field.Name()) + } + for _, field := range result.Document.Fields { + s.getOrDefineField(field.Name()) + } + } + + sort.Strings(s.FieldsInv[1:]) // keep _id as first field + + for fieldID, fieldName := range s.FieldsInv { + s.FieldsMap[fieldName] = uint16(fieldID + 1) + } + + if cap(s.IncludeDocValues) >= len(s.FieldsInv) { + s.IncludeDocValues = s.IncludeDocValues[:len(s.FieldsInv)] + } else { + s.IncludeDocValues = make([]bool, len(s.FieldsInv)) + } + + s.prepareDicts() + + for _, dict := range s.DictKeys { + sort.Strings(dict) + } + + s.processDocuments() + + storedIndexOffset, err := s.writeStoredFields() + if err != nil { + return 0, 0, 0, nil, err + } + + var fdvIndexOffset uint64 + var dictOffsets []uint64 + + if len(s.results) > 0 { + fdvIndexOffset, dictOffsets, err = s.writeDicts() + if err != nil { + return 0, 0, 0, nil, err + } + } else { + dictOffsets = make([]uint64, len(s.FieldsInv)) + } + + fieldsIndexOffset, err := persistFields(s.FieldsInv, s.w, dictOffsets) + if err != nil { + return 0, 0, 0, nil, err + } + + return storedIndexOffset, fieldsIndexOffset, fdvIndexOffset, dictOffsets, nil +} + +func (s *interim) getOrDefineField(fieldName string) int { + fieldIDPlus1, exists := s.FieldsMap[fieldName] + if !exists { + fieldIDPlus1 = uint16(len(s.FieldsInv) + 1) + s.FieldsMap[fieldName] = fieldIDPlus1 + s.FieldsInv = append(s.FieldsInv, fieldName) + + s.Dicts = append(s.Dicts, make(map[string]uint64)) + + n := len(s.DictKeys) + if n < cap(s.DictKeys) { + s.DictKeys = s.DictKeys[:n+1] + s.DictKeys[n] = s.DictKeys[n][:0] + } else { + s.DictKeys = append(s.DictKeys, []string(nil)) + } + } + + return int(fieldIDPlus1 - 1) +} + +// fill Dicts and DictKeys from analysis results +func (s *interim) prepareDicts() { + var pidNext int + + var totTFs int + var totLocs int + + visitField := func(fieldID uint16, tfs analysis.TokenFrequencies) { + dict := s.Dicts[fieldID] + dictKeys := s.DictKeys[fieldID] + + for term, tf := range tfs { + pidPlus1, exists := dict[term] + if !exists { + pidNext++ + pidPlus1 = uint64(pidNext) + + dict[term] = pidPlus1 + dictKeys = append(dictKeys, term) + + s.numTermsPerPostingsList = append(s.numTermsPerPostingsList, 0) + s.numLocsPerPostingsList = append(s.numLocsPerPostingsList, 0) + } + + pid := pidPlus1 - 1 + + s.numTermsPerPostingsList[pid] += 1 + s.numLocsPerPostingsList[pid] += len(tf.Locations) + + totLocs += len(tf.Locations) + } + + totTFs += len(tfs) + + s.DictKeys[fieldID] = dictKeys + } + + for _, result := range s.results { + // walk each composite field + for _, field := range result.Document.CompositeFields { + fieldID := uint16(s.getOrDefineField(field.Name())) + _, tf := field.Analyze() + visitField(fieldID, tf) + } + + // walk each field + for i, field := range result.Document.Fields { + fieldID := uint16(s.getOrDefineField(field.Name())) + tf := result.Analyzed[i] + visitField(fieldID, tf) + } + } + + numPostingsLists := pidNext + + if cap(s.Postings) >= numPostingsLists { + s.Postings = s.Postings[:numPostingsLists] + } else { + postings := make([]*roaring.Bitmap, numPostingsLists) + copy(postings, s.Postings[:cap(s.Postings)]) + for i := 0; i < numPostingsLists; i++ { + if postings[i] == nil { + postings[i] = roaring.New() + } + } + s.Postings = postings + } + + if cap(s.FreqNorms) >= numPostingsLists { + s.FreqNorms = s.FreqNorms[:numPostingsLists] + } else { + s.FreqNorms = make([][]interimFreqNorm, numPostingsLists) + } + + if cap(s.freqNormsBacking) >= totTFs { + s.freqNormsBacking = s.freqNormsBacking[:totTFs] + } else { + s.freqNormsBacking = make([]interimFreqNorm, totTFs) + } + + freqNormsBacking := s.freqNormsBacking + for pid, numTerms := range s.numTermsPerPostingsList { + s.FreqNorms[pid] = freqNormsBacking[0:0] + freqNormsBacking = freqNormsBacking[numTerms:] + } + + if cap(s.Locs) >= numPostingsLists { + s.Locs = s.Locs[:numPostingsLists] + } else { + s.Locs = make([][]interimLoc, numPostingsLists) + } + + if cap(s.locsBacking) >= totLocs { + s.locsBacking = s.locsBacking[:totLocs] + } else { + s.locsBacking = make([]interimLoc, totLocs) + } + + locsBacking := s.locsBacking + for pid, numLocs := range s.numLocsPerPostingsList { + s.Locs[pid] = locsBacking[0:0] + locsBacking = locsBacking[numLocs:] + } +} + +func (s *interim) processDocuments() { + numFields := len(s.FieldsInv) + reuseFieldLens := make([]int, numFields) + reuseFieldTFs := make([]analysis.TokenFrequencies, numFields) + + for docNum, result := range s.results { + for i := 0; i < numFields; i++ { // clear these for reuse + reuseFieldLens[i] = 0 + reuseFieldTFs[i] = nil + } + + s.processDocument(uint64(docNum), result, + reuseFieldLens, reuseFieldTFs) + } +} + +func (s *interim) processDocument(docNum uint64, + result *index.AnalysisResult, + fieldLens []int, fieldTFs []analysis.TokenFrequencies) { + visitField := func(fieldID uint16, fieldName string, + ln int, tf analysis.TokenFrequencies) { + fieldLens[fieldID] += ln + + existingFreqs := fieldTFs[fieldID] + if existingFreqs != nil { + existingFreqs.MergeAll(fieldName, tf) + } else { + fieldTFs[fieldID] = tf + } + } + + // walk each composite field + for _, field := range result.Document.CompositeFields { + fieldID := uint16(s.getOrDefineField(field.Name())) + ln, tf := field.Analyze() + visitField(fieldID, field.Name(), ln, tf) + } + + // walk each field + for i, field := range result.Document.Fields { + fieldID := uint16(s.getOrDefineField(field.Name())) + ln := result.Length[i] + tf := result.Analyzed[i] + visitField(fieldID, field.Name(), ln, tf) + } + + // now that it's been rolled up into fieldTFs, walk that + for fieldID, tfs := range fieldTFs { + dict := s.Dicts[fieldID] + norm := float32(1.0 / math.Sqrt(float64(fieldLens[fieldID]))) + + for term, tf := range tfs { + pid := dict[term] - 1 + bs := s.Postings[pid] + bs.Add(uint32(docNum)) + + s.FreqNorms[pid] = append(s.FreqNorms[pid], + interimFreqNorm{ + freq: uint64(tf.Frequency()), + norm: norm, + numLocs: len(tf.Locations), + }) + + if len(tf.Locations) > 0 { + locs := s.Locs[pid] + + for _, loc := range tf.Locations { + var locf = uint16(fieldID) + if loc.Field != "" { + locf = uint16(s.getOrDefineField(loc.Field)) + } + var arrayposs []uint64 + if len(loc.ArrayPositions) > 0 { + arrayposs = loc.ArrayPositions + } + locs = append(locs, interimLoc{ + fieldID: locf, + pos: uint64(loc.Position), + start: uint64(loc.Start), + end: uint64(loc.End), + arrayposs: arrayposs, + }) + } + + s.Locs[pid] = locs + } + } + } +} + +func (s *interim) writeStoredFields() ( + storedIndexOffset uint64, err error) { + varBuf := make([]byte, binary.MaxVarintLen64) + metaEncode := func(val uint64) (int, error) { + wb := binary.PutUvarint(varBuf, val) + return s.metaBuf.Write(varBuf[:wb]) + } + + data, compressed := s.tmp0[:0], s.tmp1[:0] + defer func() { s.tmp0, s.tmp1 = data, compressed }() + + // keyed by docNum + docStoredOffsets := make([]uint64, len(s.results)) + + // keyed by fieldID, for the current doc in the loop + docStoredFields := map[uint16]interimStoredField{} + + for docNum, result := range s.results { + for fieldID := range docStoredFields { // reset for next doc + delete(docStoredFields, fieldID) + } + + for _, field := range result.Document.Fields { + fieldID := uint16(s.getOrDefineField(field.Name())) + + opts := field.Options() + + if opts.IsStored() { + isf := docStoredFields[fieldID] + isf.vals = append(isf.vals, field.Value()) + isf.typs = append(isf.typs, encodeFieldType(field)) + isf.arrayposs = append(isf.arrayposs, field.ArrayPositions()) + docStoredFields[fieldID] = isf + } + + if opts.IncludeDocValues() { + s.IncludeDocValues[fieldID] = true + } + } + + var curr int + + s.metaBuf.Reset() + data = data[:0] + + // _id field special case optimizes ExternalID() lookups + idFieldVal := docStoredFields[uint16(0)].vals[0] + _, err = metaEncode(uint64(len(idFieldVal))) + if err != nil { + return 0, err + } + + // handle non-"_id" fields + for fieldID := 1; fieldID < len(s.FieldsInv); fieldID++ { + isf, exists := docStoredFields[uint16(fieldID)] + if exists { + curr, data, err = persistStoredFieldValues( + fieldID, isf.vals, isf.typs, isf.arrayposs, + curr, metaEncode, data) + if err != nil { + return 0, err + } + } + } + + metaBytes := s.metaBuf.Bytes() + + compressed = snappy.Encode(compressed[:cap(compressed)], data) + + docStoredOffsets[docNum] = uint64(s.w.Count()) + + _, err := writeUvarints(s.w, + uint64(len(metaBytes)), + uint64(len(idFieldVal)+len(compressed))) + if err != nil { + return 0, err + } + + _, err = s.w.Write(metaBytes) + if err != nil { + return 0, err + } + + _, err = s.w.Write(idFieldVal) + if err != nil { + return 0, err + } + + _, err = s.w.Write(compressed) + if err != nil { + return 0, err + } + } + + storedIndexOffset = uint64(s.w.Count()) + + for _, docStoredOffset := range docStoredOffsets { + err = binary.Write(s.w, binary.BigEndian, docStoredOffset) + if err != nil { + return 0, err + } + } + + return storedIndexOffset, nil +} + +func (s *interim) writeDicts() (fdvIndexOffset uint64, dictOffsets []uint64, err error) { + dictOffsets = make([]uint64, len(s.FieldsInv)) + + fdvOffsetsStart := make([]uint64, len(s.FieldsInv)) + fdvOffsetsEnd := make([]uint64, len(s.FieldsInv)) + + buf := s.grabBuf(binary.MaxVarintLen64) + + tfEncoder := newChunkedIntCoder(uint64(s.chunkFactor), uint64(len(s.results)-1)) + locEncoder := newChunkedIntCoder(uint64(s.chunkFactor), uint64(len(s.results)-1)) + fdvEncoder := newChunkedContentCoder(uint64(s.chunkFactor), uint64(len(s.results)-1), s.w, false) + + var docTermMap [][]byte + + if s.builder == nil { + s.builder, err = vellum.New(&s.builderBuf, nil) + if err != nil { + return 0, nil, err + } + } + + for fieldID, terms := range s.DictKeys { + if cap(docTermMap) < len(s.results) { + docTermMap = make([][]byte, len(s.results)) + } else { + docTermMap = docTermMap[0:len(s.results)] + for docNum := range docTermMap { // reset the docTermMap + docTermMap[docNum] = docTermMap[docNum][:0] + } + } + + dict := s.Dicts[fieldID] + + for _, term := range terms { // terms are already sorted + pid := dict[term] - 1 + + postingsBS := s.Postings[pid] + + freqNorms := s.FreqNorms[pid] + freqNormOffset := 0 + + locs := s.Locs[pid] + locOffset := 0 + + postingsItr := postingsBS.Iterator() + for postingsItr.HasNext() { + docNum := uint64(postingsItr.Next()) + + freqNorm := freqNorms[freqNormOffset] + + err = tfEncoder.Add(docNum, + encodeFreqHasLocs(freqNorm.freq, freqNorm.numLocs > 0), + uint64(math.Float32bits(freqNorm.norm))) + if err != nil { + return 0, nil, err + } + + if freqNorm.numLocs > 0 { + numBytesLocs := 0 + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + numBytesLocs += totalUvarintBytes( + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs)), loc.arrayposs) + } + + err = locEncoder.Add(docNum, uint64(numBytesLocs)) + if err != nil { + return 0, nil, err + } + + for _, loc := range locs[locOffset : locOffset+freqNorm.numLocs] { + err = locEncoder.Add(docNum, + uint64(loc.fieldID), loc.pos, loc.start, loc.end, + uint64(len(loc.arrayposs))) + if err != nil { + return 0, nil, err + } + + err = locEncoder.Add(docNum, loc.arrayposs...) + if err != nil { + return 0, nil, err + } + } + + locOffset += freqNorm.numLocs + } + + freqNormOffset++ + + docTermMap[docNum] = append( + append(docTermMap[docNum], term...), + termSeparator) + } + + tfEncoder.Close() + locEncoder.Close() + + postingsOffset, err := + writePostings(postingsBS, tfEncoder, locEncoder, nil, s.w, buf) + if err != nil { + return 0, nil, err + } + + if postingsOffset > uint64(0) { + err = s.builder.Insert([]byte(term), postingsOffset) + if err != nil { + return 0, nil, err + } + } + + tfEncoder.Reset() + locEncoder.Reset() + } + + err = s.builder.Close() + if err != nil { + return 0, nil, err + } + + // record where this dictionary starts + dictOffsets[fieldID] = uint64(s.w.Count()) + + vellumData := s.builderBuf.Bytes() + + // write out the length of the vellum data + n := binary.PutUvarint(buf, uint64(len(vellumData))) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + + // write this vellum to disk + _, err = s.w.Write(vellumData) + if err != nil { + return 0, nil, err + } + + // reset vellum for reuse + s.builderBuf.Reset() + + err = s.builder.Reset(&s.builderBuf) + if err != nil { + return 0, nil, err + } + + // write the field doc values + if s.IncludeDocValues[fieldID] { + for docNum, docTerms := range docTermMap { + if len(docTerms) > 0 { + err = fdvEncoder.Add(uint64(docNum), docTerms) + if err != nil { + return 0, nil, err + } + } + } + err = fdvEncoder.Close() + if err != nil { + return 0, nil, err + } + + fdvOffsetsStart[fieldID] = uint64(s.w.Count()) + + _, err = fdvEncoder.Write() + if err != nil { + return 0, nil, err + } + + fdvOffsetsEnd[fieldID] = uint64(s.w.Count()) + + fdvEncoder.Reset() + } else { + fdvOffsetsStart[fieldID] = fieldNotUninverted + fdvOffsetsEnd[fieldID] = fieldNotUninverted + } + } + + fdvIndexOffset = uint64(s.w.Count()) + + for i := 0; i < len(fdvOffsetsStart); i++ { + n := binary.PutUvarint(buf, fdvOffsetsStart[i]) + _, err := s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + n = binary.PutUvarint(buf, fdvOffsetsEnd[i]) + _, err = s.w.Write(buf[:n]) + if err != nil { + return 0, nil, err + } + } + + return fdvIndexOffset, dictOffsets, nil +} + +func encodeFieldType(f document.Field) byte { + fieldType := byte('x') + switch f.(type) { + case *document.TextField: + fieldType = 't' + case *document.NumericField: + fieldType = 'n' + case *document.DateTimeField: + fieldType = 'd' + case *document.BooleanField: + fieldType = 'b' + case *document.GeoPointField: + fieldType = 'g' + case *document.CompositeField: + fieldType = 'c' + } + return fieldType +} + +// returns the total # of bytes needed to encode the given uint64's +// into binary.PutUVarint() encoding +func totalUvarintBytes(a, b, c, d, e uint64, more []uint64) (n int) { + n = numUvarintBytes(a) + n += numUvarintBytes(b) + n += numUvarintBytes(c) + n += numUvarintBytes(d) + n += numUvarintBytes(e) + for _, v := range more { + n += numUvarintBytes(v) + } + return n +} + +// returns # of bytes needed to encode x in binary.PutUvarint() encoding +func numUvarintBytes(x uint64) (n int) { + for x >= 0x80 { + x >>= 7 + n++ + } + return n + 1 +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/posting.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/posting.go new file mode 100644 index 00000000..7bfbe0be --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/posting.go @@ -0,0 +1,790 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + "reflect" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/size" +) + +var reflectStaticSizePostingsList int +var reflectStaticSizePostingsIterator int +var reflectStaticSizePosting int +var reflectStaticSizeLocation int + +func init() { + var pl PostingsList + reflectStaticSizePostingsList = int(reflect.TypeOf(pl).Size()) + var pi PostingsIterator + reflectStaticSizePostingsIterator = int(reflect.TypeOf(pi).Size()) + var p Posting + reflectStaticSizePosting = int(reflect.TypeOf(p).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + +// FST or vellum value (uint64) encoding is determined by the top two +// highest-order or most significant bits... +// +// encoding : MSB +// name : 63 62 61...to...bit #0 (LSB) +// ----------+---+---+--------------------------------------------------- +// general : 0 | 0 | 62-bits of postingsOffset. +// ~ : 0 | 1 | reserved for future. +// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. +// ~ : 1 | 1 | reserved for future. +// +// Encoding "general" is able to handle all cases, where the +// postingsOffset points to more information about the postings for +// the term. +// +// Encoding "1-hit" is used to optimize a commonly seen case when a +// term has only a single hit. For example, a term in the _id field +// will have only 1 hit. The "1-hit" encoding is used for a term +// in a field when... +// +// - term vector info is disabled for that field; +// - and, the term appears in only a single doc for that field; +// - and, the term's freq is exactly 1 in that single doc for that field; +// - and, the docNum must fit into 31-bits; +// +// Otherwise, the "general" encoding is used instead. +// +// In the "1-hit" encoding, the field in that single doc may have +// other terms, which is supported in the "1-hit" encoding by the +// positive float31 norm. + +const FSTValEncodingMask = uint64(0xc000000000000000) +const FSTValEncodingGeneral = uint64(0x0000000000000000) +const FSTValEncoding1Hit = uint64(0x8000000000000000) + +func FSTValEncode1Hit(docNum uint64, normBits uint64) uint64 { + return FSTValEncoding1Hit | ((mask31Bits & normBits) << 31) | (mask31Bits & docNum) +} + +func FSTValDecode1Hit(v uint64) (docNum uint64, normBits uint64) { + return (mask31Bits & v), (mask31Bits & (v >> 31)) +} + +const mask31Bits = uint64(0x000000007fffffff) + +func under32Bits(x uint64) bool { + return x <= mask31Bits +} + +const docNum1HitFinished = math.MaxUint64 + +// PostingsList is an in-memory represenation of a postings list +type PostingsList struct { + sb *SegmentBase + postingsOffset uint64 + freqOffset uint64 + locOffset uint64 + postings *roaring.Bitmap + except *roaring.Bitmap + + // when normBits1Hit != 0, then this postings list came from a + // 1-hit encoding, and only the docNum1Hit & normBits1Hit apply + docNum1Hit uint64 + normBits1Hit uint64 +} + +// represents an immutable, empty postings list +var emptyPostingsList = &PostingsList{} + +func (p *PostingsList) Size() int { + sizeInBytes := reflectStaticSizePostingsList + size.SizeOfPtr + + if p.except != nil { + sizeInBytes += int(p.except.GetSizeInBytes()) + } + + return sizeInBytes +} + +func (p *PostingsList) OrInto(receiver *roaring.Bitmap) { + if p.normBits1Hit != 0 { + receiver.Add(uint32(p.docNum1Hit)) + return + } + + if p.postings != nil { + receiver.Or(p.postings) + } +} + +// Iterator returns an iterator for this postings list +func (p *PostingsList) Iterator(includeFreq, includeNorm, includeLocs bool, + prealloc segment.PostingsIterator) segment.PostingsIterator { + if p.normBits1Hit == 0 && p.postings == nil { + return emptyPostingsIterator + } + + var preallocPI *PostingsIterator + pi, ok := prealloc.(*PostingsIterator) + if ok && pi != nil { + preallocPI = pi + } + if preallocPI == emptyPostingsIterator { + preallocPI = nil + } + + return p.iterator(includeFreq, includeNorm, includeLocs, preallocPI) +} + +func (p *PostingsList) iterator(includeFreq, includeNorm, includeLocs bool, + rv *PostingsIterator) *PostingsIterator { + if rv == nil { + rv = &PostingsIterator{} + } else { + freqNormReader := rv.freqNormReader + if freqNormReader != nil { + freqNormReader.Reset([]byte(nil)) + } + + locReader := rv.locReader + if locReader != nil { + locReader.Reset([]byte(nil)) + } + + freqChunkOffsets := rv.freqChunkOffsets[:0] + locChunkOffsets := rv.locChunkOffsets[:0] + + nextLocs := rv.nextLocs[:0] + nextSegmentLocs := rv.nextSegmentLocs[:0] + + buf := rv.buf + + *rv = PostingsIterator{} // clear the struct + + rv.freqNormReader = freqNormReader + rv.locReader = locReader + + rv.freqChunkOffsets = freqChunkOffsets + rv.locChunkOffsets = locChunkOffsets + + rv.nextLocs = nextLocs + rv.nextSegmentLocs = nextSegmentLocs + + rv.buf = buf + } + rv.postings = p + + if p.normBits1Hit != 0 { + // "1-hit" encoding + rv.docNum1Hit = p.docNum1Hit + rv.normBits1Hit = p.normBits1Hit + + if p.except != nil && p.except.Contains(uint32(rv.docNum1Hit)) { + rv.docNum1Hit = docNum1HitFinished + } + + return rv + } + + // "general" encoding, check if empty + if p.postings == nil { + return rv + } + + var n uint64 + var read int + + // prepare the freq chunk details + rv.includeFreqNorm = includeFreq || includeNorm + if rv.includeFreqNorm { + var numFreqChunks uint64 + numFreqChunks, read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + if cap(rv.freqChunkOffsets) >= int(numFreqChunks) { + rv.freqChunkOffsets = rv.freqChunkOffsets[:int(numFreqChunks)] + } else { + rv.freqChunkOffsets = make([]uint64, int(numFreqChunks)) + } + for i := 0; i < int(numFreqChunks); i++ { + rv.freqChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.freqOffset+n : p.freqOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + } + rv.freqChunkStart = p.freqOffset + n + } + + // prepare the loc chunk details + rv.includeLocs = includeLocs + if rv.includeLocs { + n = 0 + var numLocChunks uint64 + numLocChunks, read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + if cap(rv.locChunkOffsets) >= int(numLocChunks) { + rv.locChunkOffsets = rv.locChunkOffsets[:int(numLocChunks)] + } else { + rv.locChunkOffsets = make([]uint64, int(numLocChunks)) + } + for i := 0; i < int(numLocChunks); i++ { + rv.locChunkOffsets[i], read = binary.Uvarint(p.sb.mem[p.locOffset+n : p.locOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + } + rv.locChunkStart = p.locOffset + n + } + + rv.all = p.postings.Iterator() + if p.except != nil { + rv.ActualBM = roaring.AndNot(p.postings, p.except) + rv.Actual = rv.ActualBM.Iterator() + } else { + rv.ActualBM = p.postings + rv.Actual = p.postings.Iterator() + } + + return rv +} + +// Count returns the number of items on this postings list +func (p *PostingsList) Count() uint64 { + var n uint64 + if p.normBits1Hit != 0 { + n = 1 + } else if p.postings != nil { + n = p.postings.GetCardinality() + } + var e uint64 + if p.except != nil { + e = p.except.GetCardinality() + } + if n <= e { + return 0 + } + return n - e +} + +func (rv *PostingsList) read(postingsOffset uint64, d *Dictionary) error { + rv.postingsOffset = postingsOffset + + // handle "1-hit" encoding special case + if rv.postingsOffset&FSTValEncodingMask == FSTValEncoding1Hit { + return rv.init1Hit(postingsOffset) + } + + // read the location of the freq/norm details + var n uint64 + var read int + + rv.freqOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+binary.MaxVarintLen64]) + n += uint64(read) + + rv.locOffset, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + var postingsLen uint64 + postingsLen, read = binary.Uvarint(d.sb.mem[postingsOffset+n : postingsOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + roaringBytes := d.sb.mem[postingsOffset+n : postingsOffset+n+postingsLen] + + if rv.postings == nil { + rv.postings = roaring.NewBitmap() + } + _, err := rv.postings.FromBuffer(roaringBytes) + if err != nil { + return fmt.Errorf("error loading roaring bitmap: %v", err) + } + + return nil +} + +func (rv *PostingsList) init1Hit(fstVal uint64) error { + docNum, normBits := FSTValDecode1Hit(fstVal) + + rv.docNum1Hit = docNum + rv.normBits1Hit = normBits + + return nil +} + +// PostingsIterator provides a way to iterate through the postings list +type PostingsIterator struct { + postings *PostingsList + all roaring.IntIterable + Actual roaring.IntIterable + ActualBM *roaring.Bitmap + + currChunk uint32 + currChunkFreqNorm []byte + currChunkLoc []byte + + freqNormReader *bytes.Reader + locReader *bytes.Reader + + freqChunkOffsets []uint64 + freqChunkStart uint64 + + locChunkOffsets []uint64 + locChunkStart uint64 + + next Posting // reused across Next() calls + nextLocs []Location // reused across Next() calls + nextSegmentLocs []segment.Location // reused across Next() calls + + docNum1Hit uint64 + normBits1Hit uint64 + + buf []byte + + includeFreqNorm bool + includeLocs bool +} + +var emptyPostingsIterator = &PostingsIterator{} + +func (i *PostingsIterator) Size() int { + sizeInBytes := reflectStaticSizePostingsIterator + size.SizeOfPtr + + len(i.currChunkFreqNorm) + + len(i.currChunkLoc) + + len(i.freqChunkOffsets)*size.SizeOfUint64 + + len(i.locChunkOffsets)*size.SizeOfUint64 + + i.next.Size() + + for _, entry := range i.nextLocs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +func (i *PostingsIterator) loadChunk(chunk int) error { + if i.includeFreqNorm { + if chunk >= len(i.freqChunkOffsets) { + return fmt.Errorf("tried to load freq chunk that doesn't exist %d/(%d)", + chunk, len(i.freqChunkOffsets)) + } + + end, start := i.freqChunkStart, i.freqChunkStart + s, e := readChunkBoundary(chunk, i.freqChunkOffsets) + start += s + end += e + i.currChunkFreqNorm = i.postings.sb.mem[start:end] + if i.freqNormReader == nil { + i.freqNormReader = bytes.NewReader(i.currChunkFreqNorm) + } else { + i.freqNormReader.Reset(i.currChunkFreqNorm) + } + } + + if i.includeLocs { + if chunk >= len(i.locChunkOffsets) { + return fmt.Errorf("tried to load loc chunk that doesn't exist %d/(%d)", + chunk, len(i.locChunkOffsets)) + } + + end, start := i.locChunkStart, i.locChunkStart + s, e := readChunkBoundary(chunk, i.locChunkOffsets) + start += s + end += e + i.currChunkLoc = i.postings.sb.mem[start:end] + if i.locReader == nil { + i.locReader = bytes.NewReader(i.currChunkLoc) + } else { + i.locReader.Reset(i.currChunkLoc) + } + } + + i.currChunk = uint32(chunk) + return nil +} + +func (i *PostingsIterator) readFreqNormHasLocs() (uint64, uint64, bool, error) { + if i.normBits1Hit != 0 { + return 1, i.normBits1Hit, false, nil + } + + freqHasLocs, err := binary.ReadUvarint(i.freqNormReader) + if err != nil { + return 0, 0, false, fmt.Errorf("error reading frequency: %v", err) + } + freq, hasLocs := decodeFreqHasLocs(freqHasLocs) + + normBits, err := binary.ReadUvarint(i.freqNormReader) + if err != nil { + return 0, 0, false, fmt.Errorf("error reading norm: %v", err) + } + + return freq, normBits, hasLocs, err +} + +func encodeFreqHasLocs(freq uint64, hasLocs bool) uint64 { + rv := freq << 1 + if hasLocs { + rv = rv | 0x01 // 0'th LSB encodes whether there are locations + } + return rv +} + +func decodeFreqHasLocs(freqHasLocs uint64) (uint64, bool) { + freq := freqHasLocs >> 1 + hasLocs := freqHasLocs&0x01 != 0 + return freq, hasLocs +} + +// readLocation processes all the integers on the stream representing a single +// location. if you care about it, pass in a non-nil location struct, and we +// will fill it. if you don't care about it, pass in nil and we safely consume +// the contents. +func (i *PostingsIterator) readLocation(l *Location) error { + // read off field + fieldID, err := binary.ReadUvarint(i.locReader) + if err != nil { + return fmt.Errorf("error reading location field: %v", err) + } + // read off pos + pos, err := binary.ReadUvarint(i.locReader) + if err != nil { + return fmt.Errorf("error reading location pos: %v", err) + } + // read off start + start, err := binary.ReadUvarint(i.locReader) + if err != nil { + return fmt.Errorf("error reading location start: %v", err) + } + // read off end + end, err := binary.ReadUvarint(i.locReader) + if err != nil { + return fmt.Errorf("error reading location end: %v", err) + } + // read off num array pos + numArrayPos, err := binary.ReadUvarint(i.locReader) + if err != nil { + return fmt.Errorf("error reading location num array pos: %v", err) + } + + // group these together for less branching + if l != nil { + l.field = i.postings.sb.fieldsInv[fieldID] + l.pos = pos + l.start = start + l.end = end + if cap(l.ap) < int(numArrayPos) { + l.ap = make([]uint64, int(numArrayPos)) + } else { + l.ap = l.ap[:int(numArrayPos)] + } + } + + // read off array positions + for k := 0; k < int(numArrayPos); k++ { + ap, err := binary.ReadUvarint(i.locReader) + if err != nil { + return fmt.Errorf("error reading array position: %v", err) + } + if l != nil { + l.ap[k] = ap + } + } + + return nil +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) Next() (segment.Posting, error) { + return i.nextAtOrAfter(0) +} + +// Advance returns the posting at the specified docNum or it is not present +// the next posting, or if the end is reached, nil +func (i *PostingsIterator) Advance(docNum uint64) (segment.Posting, error) { + return i.nextAtOrAfter(docNum) +} + +// Next returns the next posting on the postings list, or nil at the end +func (i *PostingsIterator) nextAtOrAfter(atOrAfter uint64) (segment.Posting, error) { + docNum, exists, err := i.nextDocNumAtOrAfter(atOrAfter) + if err != nil || !exists { + return nil, err + } + + i.next = Posting{} // clear the struct + rv := &i.next + rv.docNum = docNum + + if !i.includeFreqNorm { + return rv, nil + } + + var normBits uint64 + var hasLocs bool + + rv.freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return nil, err + } + + rv.norm = math.Float32frombits(uint32(normBits)) + + if i.includeLocs && hasLocs { + // prepare locations into reused slices, where we assume + // rv.freq >= "number of locs", since in a composite field, + // some component fields might have their IncludeTermVector + // flags disabled while other component fields are enabled + if cap(i.nextLocs) >= int(rv.freq) { + i.nextLocs = i.nextLocs[0:rv.freq] + } else { + i.nextLocs = make([]Location, rv.freq, rv.freq*2) + } + if cap(i.nextSegmentLocs) < int(rv.freq) { + i.nextSegmentLocs = make([]segment.Location, rv.freq, rv.freq*2) + } + rv.locs = i.nextSegmentLocs[:0] + + numLocsBytes, err := binary.ReadUvarint(i.locReader) + if err != nil { + return nil, fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + j := 0 + startBytesRemaining := i.locReader.Len() // # bytes remaining in the locReader + for startBytesRemaining-i.locReader.Len() < int(numLocsBytes) { + err := i.readLocation(&i.nextLocs[j]) + if err != nil { + return nil, err + } + rv.locs = append(rv.locs, &i.nextLocs[j]) + j++ + } + } + + return rv, nil +} + +var freqHasLocs1Hit = encodeFreqHasLocs(1, false) + +// nextBytes returns the docNum and the encoded freq & loc bytes for +// the next posting +func (i *PostingsIterator) nextBytes() ( + docNumOut uint64, freq uint64, normBits uint64, + bytesFreqNorm []byte, bytesLoc []byte, err error) { + docNum, exists, err := i.nextDocNumAtOrAfter(0) + if err != nil || !exists { + return 0, 0, 0, nil, nil, err + } + + if i.normBits1Hit != 0 { + if i.buf == nil { + i.buf = make([]byte, binary.MaxVarintLen64*2) + } + n := binary.PutUvarint(i.buf, freqHasLocs1Hit) + n += binary.PutUvarint(i.buf[n:], i.normBits1Hit) + return docNum, uint64(1), i.normBits1Hit, i.buf[:n], nil, nil + } + + startFreqNorm := len(i.currChunkFreqNorm) - i.freqNormReader.Len() + + var hasLocs bool + + freq, normBits, hasLocs, err = i.readFreqNormHasLocs() + if err != nil { + return 0, 0, 0, nil, nil, err + } + + endFreqNorm := len(i.currChunkFreqNorm) - i.freqNormReader.Len() + bytesFreqNorm = i.currChunkFreqNorm[startFreqNorm:endFreqNorm] + + if hasLocs { + startLoc := len(i.currChunkLoc) - i.locReader.Len() + + numLocsBytes, err := binary.ReadUvarint(i.locReader) + if err != nil { + return 0, 0, 0, nil, nil, + fmt.Errorf("error reading location nextBytes numLocs: %v", err) + } + + // skip over all the location bytes + _, err = i.locReader.Seek(int64(numLocsBytes), io.SeekCurrent) + if err != nil { + return 0, 0, 0, nil, nil, err + } + + endLoc := len(i.currChunkLoc) - i.locReader.Len() + bytesLoc = i.currChunkLoc[startLoc:endLoc] + } + + return docNum, freq, normBits, bytesFreqNorm, bytesLoc, nil +} + +// nextDocNum returns the next docNum on the postings list, and also +// sets up the currChunk / loc related fields of the iterator. +func (i *PostingsIterator) nextDocNumAtOrAfter(atOrAfter uint64) (uint64, bool, error) { + if i.normBits1Hit != 0 { + if i.docNum1Hit == docNum1HitFinished { + return 0, false, nil + } + if i.docNum1Hit < atOrAfter { + // advanced past our 1-hit + i.docNum1Hit = docNum1HitFinished // consume our 1-hit docNum + return 0, false, nil + } + docNum := i.docNum1Hit + i.docNum1Hit = docNum1HitFinished // consume our 1-hit docNum + return docNum, true, nil + } + + if i.Actual == nil || !i.Actual.HasNext() { + return 0, false, nil + } + + n := i.Actual.Next() + for uint64(n) < atOrAfter && i.Actual.HasNext() { + n = i.Actual.Next() + } + if uint64(n) < atOrAfter { + // couldn't find anything + return 0, false, nil + } + allN := i.all.Next() + + nChunk := n / i.postings.sb.chunkFactor + allNChunk := allN / i.postings.sb.chunkFactor + + // n is the next actual hit (excluding some postings), and + // allN is the next hit in the full postings, and + // if they don't match, move 'all' forwards until they do + for allN != n { + // in the same chunk, so move the freq/norm/loc decoders forward + if i.includeFreqNorm && allNChunk == nChunk { + if i.currChunk != nChunk || i.currChunkFreqNorm == nil { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + // read off freq/offsets even though we don't care about them + _, _, hasLocs, err := i.readFreqNormHasLocs() + if err != nil { + return 0, false, err + } + + if i.includeLocs && hasLocs { + numLocsBytes, err := binary.ReadUvarint(i.locReader) + if err != nil { + return 0, false, fmt.Errorf("error reading location numLocsBytes: %v", err) + } + + // skip over all the location bytes + _, err = i.locReader.Seek(int64(numLocsBytes), io.SeekCurrent) + if err != nil { + return 0, false, err + } + } + } + + allN = i.all.Next() + allNChunk = allN / i.postings.sb.chunkFactor + } + + if i.includeFreqNorm && (i.currChunk != nChunk || i.currChunkFreqNorm == nil) { + err := i.loadChunk(int(nChunk)) + if err != nil { + return 0, false, fmt.Errorf("error loading chunk: %v", err) + } + } + + return uint64(n), true, nil +} + +// Posting is a single entry in a postings list +type Posting struct { + docNum uint64 + freq uint64 + norm float32 + locs []segment.Location +} + +func (p *Posting) Size() int { + sizeInBytes := reflectStaticSizePosting + + for _, entry := range p.locs { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + +// Number returns the document number of this posting in this segment +func (p *Posting) Number() uint64 { + return p.docNum +} + +// Frequency returns the frequence of occurance of this term in this doc/field +func (p *Posting) Frequency() uint64 { + return p.freq +} + +// Norm returns the normalization factor for this posting +func (p *Posting) Norm() float64 { + return float64(p.norm) +} + +// Locations returns the location information for each occurance +func (p *Posting) Locations() []segment.Location { + return p.locs +} + +// Location represents the location of a single occurance +type Location struct { + field string + pos uint64 + start uint64 + end uint64 + ap []uint64 +} + +func (l *Location) Size() int { + return reflectStaticSizeLocation + + len(l.field) + + len(l.ap)*size.SizeOfUint64 +} + +// Field returns the name of the field (useful in composite fields to know +// which original field the value came from) +func (l *Location) Field() string { + return l.field +} + +// Start returns the start byte offset of this occurance +func (l *Location) Start() uint64 { + return l.start +} + +// End returns the end byte offset of this occurance +func (l *Location) End() uint64 { + return l.end +} + +// Pos returns the 1-based phrase position of this occurance +func (l *Location) Pos() uint64 { + return l.pos +} + +// ArrayPositions returns the array position vector associated with this occurance +func (l *Location) ArrayPositions() []uint64 { + return l.ap +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/read.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/read.go new file mode 100644 index 00000000..e47d4c6a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/read.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import "encoding/binary" + +func (s *SegmentBase) getDocStoredMetaAndCompressed(docNum uint64) ([]byte, []byte) { + _, storedOffset, n, metaLen, dataLen := s.getDocStoredOffsets(docNum) + + meta := s.mem[storedOffset+n : storedOffset+n+metaLen] + data := s.mem[storedOffset+n+metaLen : storedOffset+n+metaLen+dataLen] + + return meta, data +} + +func (s *SegmentBase) getDocStoredOffsets(docNum uint64) ( + uint64, uint64, uint64, uint64, uint64) { + indexOffset := s.storedIndexOffset + (8 * docNum) + + storedOffset := binary.BigEndian.Uint64(s.mem[indexOffset : indexOffset+8]) + + var n uint64 + + metaLen, read := binary.Uvarint(s.mem[storedOffset : storedOffset+binary.MaxVarintLen64]) + n += uint64(read) + + dataLen, read := binary.Uvarint(s.mem[storedOffset+n : storedOffset+n+binary.MaxVarintLen64]) + n += uint64(read) + + return indexOffset, storedOffset, n, metaLen, dataLen +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/segment.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/segment.go new file mode 100644 index 00000000..5f9a562f --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/segment.go @@ -0,0 +1,534 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "reflect" + "sync" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/size" + "github.com/couchbase/vellum" + mmap "github.com/edsrzf/mmap-go" + "github.com/golang/snappy" +) + +var reflectStaticSizeSegmentBase int + +func init() { + var sb SegmentBase + reflectStaticSizeSegmentBase = int(reflect.TypeOf(sb).Size()) +} + +// Open returns a zap impl of a segment +func Open(path string) (segment.Segment, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + mm, err := mmap.Map(f, mmap.RDONLY, 0) + if err != nil { + // mmap failed, try to close the file + _ = f.Close() + return nil, err + } + + rv := &Segment{ + SegmentBase: SegmentBase{ + mem: mm[0 : len(mm)-FooterSize], + fieldsMap: make(map[string]uint16), + fieldDvReaders: make(map[uint16]*docValueReader), + }, + f: f, + mm: mm, + path: path, + refs: 1, + } + rv.SegmentBase.updateSize() + + err = rv.loadConfig() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadFields() + if err != nil { + _ = rv.Close() + return nil, err + } + + err = rv.loadDvReaders() + if err != nil { + _ = rv.Close() + return nil, err + } + + return rv, nil +} + +// SegmentBase is a memory only, read-only implementation of the +// segment.Segment interface, using zap's data representation. +type SegmentBase struct { + mem []byte + memCRC uint32 + chunkFactor uint32 + fieldsMap map[string]uint16 // fieldName -> fieldID+1 + fieldsInv []string // fieldID -> fieldName + numDocs uint64 + storedIndexOffset uint64 + fieldsIndexOffset uint64 + docValueOffset uint64 + dictLocs []uint64 + fieldDvReaders map[uint16]*docValueReader // naive chunk cache per field + size uint64 +} + +func (sb *SegmentBase) Size() int { + return int(sb.size) +} + +func (sb *SegmentBase) updateSize() { + sizeInBytes := reflectStaticSizeSegmentBase + + cap(sb.mem) + + // fieldsMap + for k, _ := range sb.fieldsMap { + sizeInBytes += (len(k) + size.SizeOfString) + size.SizeOfUint16 + } + + // fieldsInv, dictLocs + for _, entry := range sb.fieldsInv { + sizeInBytes += len(entry) + size.SizeOfString + } + sizeInBytes += len(sb.dictLocs) * size.SizeOfUint64 + + // fieldDvReaders + for _, v := range sb.fieldDvReaders { + sizeInBytes += size.SizeOfUint16 + size.SizeOfPtr + if v != nil { + sizeInBytes += v.size() + } + } + + sb.size = uint64(sizeInBytes) +} + +func (sb *SegmentBase) AddRef() {} +func (sb *SegmentBase) DecRef() (err error) { return nil } +func (sb *SegmentBase) Close() (err error) { return nil } + +// Segment implements a persisted segment.Segment interface, by +// embedding an mmap()'ed SegmentBase. +type Segment struct { + SegmentBase + + f *os.File + mm mmap.MMap + path string + version uint32 + crc uint32 + + m sync.Mutex // Protects the fields that follow. + refs int64 +} + +func (s *Segment) Size() int { + // 8 /* size of file pointer */ + // 4 /* size of version -> uint32 */ + // 4 /* size of crc -> uint32 */ + sizeOfUints := 16 + + sizeInBytes := (len(s.path) + size.SizeOfString) + sizeOfUints + + // mutex, refs -> int64 + sizeInBytes += 16 + + // do not include the mmap'ed part + return sizeInBytes + s.SegmentBase.Size() - cap(s.mem) +} + +func (s *Segment) AddRef() { + s.m.Lock() + s.refs++ + s.m.Unlock() +} + +func (s *Segment) DecRef() (err error) { + s.m.Lock() + s.refs-- + if s.refs == 0 { + err = s.closeActual() + } + s.m.Unlock() + return err +} + +func (s *Segment) loadConfig() error { + crcOffset := len(s.mm) - 4 + s.crc = binary.BigEndian.Uint32(s.mm[crcOffset : crcOffset+4]) + + verOffset := crcOffset - 4 + s.version = binary.BigEndian.Uint32(s.mm[verOffset : verOffset+4]) + if s.version != Version { + return fmt.Errorf("unsupported version %d", s.version) + } + + chunkOffset := verOffset - 4 + s.chunkFactor = binary.BigEndian.Uint32(s.mm[chunkOffset : chunkOffset+4]) + + docValueOffset := chunkOffset - 8 + s.docValueOffset = binary.BigEndian.Uint64(s.mm[docValueOffset : docValueOffset+8]) + + fieldsIndexOffset := docValueOffset - 8 + s.fieldsIndexOffset = binary.BigEndian.Uint64(s.mm[fieldsIndexOffset : fieldsIndexOffset+8]) + + storedIndexOffset := fieldsIndexOffset - 8 + s.storedIndexOffset = binary.BigEndian.Uint64(s.mm[storedIndexOffset : storedIndexOffset+8]) + + numDocsOffset := storedIndexOffset - 8 + s.numDocs = binary.BigEndian.Uint64(s.mm[numDocsOffset : numDocsOffset+8]) + return nil +} + +func (s *SegmentBase) loadFields() error { + // NOTE for now we assume the fields index immediately preceeds + // the footer, and if this changes, need to adjust accordingly (or + // store explicit length), where s.mem was sliced from s.mm in Open(). + fieldsIndexEnd := uint64(len(s.mem)) + + // iterate through fields index + var fieldID uint64 + for s.fieldsIndexOffset+(8*fieldID) < fieldsIndexEnd { + addr := binary.BigEndian.Uint64(s.mem[s.fieldsIndexOffset+(8*fieldID) : s.fieldsIndexOffset+(8*fieldID)+8]) + + dictLoc, read := binary.Uvarint(s.mem[addr:fieldsIndexEnd]) + n := uint64(read) + s.dictLocs = append(s.dictLocs, dictLoc) + + var nameLen uint64 + nameLen, read = binary.Uvarint(s.mem[addr+n : fieldsIndexEnd]) + n += uint64(read) + + name := string(s.mem[addr+n : addr+n+nameLen]) + s.fieldsInv = append(s.fieldsInv, name) + s.fieldsMap[name] = uint16(fieldID + 1) + + fieldID++ + } + return nil +} + +// Dictionary returns the term dictionary for the specified field +func (s *SegmentBase) Dictionary(field string) (segment.TermDictionary, error) { + dict, err := s.dictionary(field) + if err == nil && dict == nil { + return &segment.EmptyDictionary{}, nil + } + return dict, err +} + +func (sb *SegmentBase) dictionary(field string) (rv *Dictionary, err error) { + fieldIDPlus1 := sb.fieldsMap[field] + if fieldIDPlus1 > 0 { + rv = &Dictionary{ + sb: sb, + field: field, + fieldID: fieldIDPlus1 - 1, + } + + dictStart := sb.dictLocs[rv.fieldID] + if dictStart > 0 { + // read the length of the vellum data + vellumLen, read := binary.Uvarint(sb.mem[dictStart : dictStart+binary.MaxVarintLen64]) + fstBytes := sb.mem[dictStart+uint64(read) : dictStart+uint64(read)+vellumLen] + if fstBytes != nil { + rv.fst, err = vellum.Load(fstBytes) + if err != nil { + return nil, fmt.Errorf("dictionary field %s vellum err: %v", field, err) + } + rv.fstReader, err = rv.fst.Reader() + if err != nil { + return nil, fmt.Errorf("dictionary field %s vellum Reader err: %v", field, err) + } + } + } + } + + return rv, nil +} + +// visitDocumentCtx holds data structures that are reusable across +// multiple VisitDocument() calls to avoid memory allocations +type visitDocumentCtx struct { + buf []byte + reader bytes.Reader + arrayPos []uint64 +} + +var visitDocumentCtxPool = sync.Pool{ + New: func() interface{} { + reuse := &visitDocumentCtx{} + return reuse + }, +} + +// VisitDocument invokes the DocFieldValueVistor for each stored field +// for the specified doc number +func (s *SegmentBase) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { + // first make sure this is a valid number in this segment + if num < s.numDocs { + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + idFieldVal := compressed[:idFieldValLen] + + keepGoing := visitor("_id", byte('t'), idFieldVal, nil) + if !keepGoing { + visitDocumentCtxPool.Put(vdc) + return nil + } + + // handle non-"_id" fields + compressed = compressed[idFieldValLen:] + + uncompressed, err := snappy.Decode(vdc.buf[:cap(vdc.buf)], compressed) + if err != nil { + return err + } + + for keepGoing { + field, err := binary.ReadUvarint(&vdc.reader) + if err == io.EOF { + break + } + if err != nil { + return err + } + typ, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + offset, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + l, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + numap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + var arrayPos []uint64 + if numap > 0 { + if cap(vdc.arrayPos) < int(numap) { + vdc.arrayPos = make([]uint64, numap) + } + arrayPos = vdc.arrayPos[:numap] + for i := 0; i < int(numap); i++ { + ap, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return err + } + arrayPos[i] = ap + } + } + + value := uncompressed[offset : offset+l] + keepGoing = visitor(s.fieldsInv[field], byte(typ), value, arrayPos) + } + + vdc.buf = uncompressed + visitDocumentCtxPool.Put(vdc) + } + return nil +} + +// DocID returns the value of the _id field for the given docNum +func (s *SegmentBase) DocID(num uint64) ([]byte, error) { + if num >= s.numDocs { + return nil, nil + } + + vdc := visitDocumentCtxPool.Get().(*visitDocumentCtx) + + meta, compressed := s.getDocStoredMetaAndCompressed(num) + + vdc.reader.Reset(meta) + + // handle _id field special case + idFieldValLen, err := binary.ReadUvarint(&vdc.reader) + if err != nil { + return nil, err + } + idFieldVal := compressed[:idFieldValLen] + + visitDocumentCtxPool.Put(vdc) + + return idFieldVal, nil +} + +// Count returns the number of documents in this segment. +func (s *SegmentBase) Count() uint64 { + return s.numDocs +} + +// DocNumbers returns a bitset corresponding to the doc numbers of all the +// provided _id strings +func (s *SegmentBase) DocNumbers(ids []string) (*roaring.Bitmap, error) { + rv := roaring.New() + + if len(s.fieldsMap) > 0 { + idDict, err := s.dictionary("_id") + if err != nil { + return nil, err + } + + postingsList := emptyPostingsList + for _, id := range ids { + postingsList, err = idDict.postingsList([]byte(id), nil, postingsList) + if err != nil { + return nil, err + } + postingsList.OrInto(rv) + } + } + + return rv, nil +} + +// Fields returns the field names used in this segment +func (s *SegmentBase) Fields() []string { + return s.fieldsInv +} + +// Path returns the path of this segment on disk +func (s *Segment) Path() string { + return s.path +} + +// Close releases all resources associated with this segment +func (s *Segment) Close() (err error) { + return s.DecRef() +} + +func (s *Segment) closeActual() (err error) { + if s.mm != nil { + err = s.mm.Unmap() + } + // try to close file even if unmap failed + if s.f != nil { + err2 := s.f.Close() + if err == nil { + // try to return first error + err = err2 + } + } + return +} + +// some helpers i started adding for the command-line utility + +// Data returns the underlying mmaped data slice +func (s *Segment) Data() []byte { + return s.mm +} + +// CRC returns the CRC value stored in the file footer +func (s *Segment) CRC() uint32 { + return s.crc +} + +// Version returns the file version in the file footer +func (s *Segment) Version() uint32 { + return s.version +} + +// ChunkFactor returns the chunk factor in the file footer +func (s *Segment) ChunkFactor() uint32 { + return s.chunkFactor +} + +// FieldsIndexOffset returns the fields index offset in the file footer +func (s *Segment) FieldsIndexOffset() uint64 { + return s.fieldsIndexOffset +} + +// StoredIndexOffset returns the stored value index offset in the file footer +func (s *Segment) StoredIndexOffset() uint64 { + return s.storedIndexOffset +} + +// DocValueOffset returns the docValue offset in the file footer +func (s *Segment) DocValueOffset() uint64 { + return s.docValueOffset +} + +// NumDocs returns the number of documents in the file footer +func (s *Segment) NumDocs() uint64 { + return s.numDocs +} + +// DictAddr is a helper function to compute the file offset where the +// dictionary is stored for the specified field. +func (s *Segment) DictAddr(field string) (uint64, error) { + fieldIDPlus1, ok := s.fieldsMap[field] + if !ok { + return 0, fmt.Errorf("no such field '%s'", field) + } + + return s.dictLocs[fieldIDPlus1-1], nil +} + +func (s *SegmentBase) loadDvReaders() error { + if s.docValueOffset == fieldNotUninverted { + return nil + } + + var read uint64 + for fieldID, field := range s.fieldsInv { + var fieldLocStart, fieldLocEnd uint64 + var n int + fieldLocStart, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset start for field %d", fieldID) + } + read += uint64(n) + fieldLocEnd, n = binary.Uvarint(s.mem[s.docValueOffset+read : s.docValueOffset+read+binary.MaxVarintLen64]) + if n <= 0 { + return fmt.Errorf("loadDvReaders: failed to read the docvalue offset end for field %d", fieldID) + } + read += uint64(n) + + s.fieldDvReaders[uint16(fieldID)], _ = s.loadFieldDocValueReader(field, fieldLocStart, fieldLocEnd) + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/write.go b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/write.go new file mode 100644 index 00000000..cddaedd0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/segment/zap/write.go @@ -0,0 +1,145 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zap + +import ( + "encoding/binary" + "io" + + "github.com/RoaringBitmap/roaring" +) + +// writes out the length of the roaring bitmap in bytes as varint +// then writes out the roaring bitmap itself +func writeRoaringWithLen(r *roaring.Bitmap, w io.Writer, + reuseBufVarint []byte) (int, error) { + buf, err := r.ToBytes() + if err != nil { + return 0, err + } + + var tw int + + // write out the length + n := binary.PutUvarint(reuseBufVarint, uint64(len(buf))) + nw, err := w.Write(reuseBufVarint[:n]) + tw += nw + if err != nil { + return tw, err + } + + // write out the roaring bytes + nw, err = w.Write(buf) + tw += nw + if err != nil { + return tw, err + } + + return tw, nil +} + +func persistFields(fieldsInv []string, w *CountHashWriter, dictLocs []uint64) (uint64, error) { + var rv uint64 + var fieldsOffsets []uint64 + + for fieldID, fieldName := range fieldsInv { + // record start of this field + fieldsOffsets = append(fieldsOffsets, uint64(w.Count())) + + // write out the dict location and field name length + _, err := writeUvarints(w, dictLocs[fieldID], uint64(len(fieldName))) + if err != nil { + return 0, err + } + + // write out the field name + _, err = w.Write([]byte(fieldName)) + if err != nil { + return 0, err + } + } + + // now write out the fields index + rv = uint64(w.Count()) + for fieldID := range fieldsInv { + err := binary.Write(w, binary.BigEndian, fieldsOffsets[fieldID]) + if err != nil { + return 0, err + } + } + + return rv, nil +} + +// FooterSize is the size of the footer record in bytes +// crc + ver + chunk + field offset + stored offset + num docs + docValueOffset +const FooterSize = 4 + 4 + 4 + 8 + 8 + 8 + 8 + +func persistFooter(numDocs, storedIndexOffset, fieldsIndexOffset, docValueOffset uint64, + chunkFactor uint32, crcBeforeFooter uint32, writerIn io.Writer) error { + w := NewCountHashWriter(writerIn) + w.crc = crcBeforeFooter + + // write out the number of docs + err := binary.Write(w, binary.BigEndian, numDocs) + if err != nil { + return err + } + // write out the stored field index location: + err = binary.Write(w, binary.BigEndian, storedIndexOffset) + if err != nil { + return err + } + // write out the field index location + err = binary.Write(w, binary.BigEndian, fieldsIndexOffset) + if err != nil { + return err + } + // write out the fieldDocValue location + err = binary.Write(w, binary.BigEndian, docValueOffset) + if err != nil { + return err + } + // write out 32-bit chunk factor + err = binary.Write(w, binary.BigEndian, chunkFactor) + if err != nil { + return err + } + // write out 32-bit version + err = binary.Write(w, binary.BigEndian, Version) + if err != nil { + return err + } + // write out CRC-32 of everything upto but not including this CRC + err = binary.Write(w, binary.BigEndian, w.crc) + if err != nil { + return err + } + return nil +} + +func writeUvarints(w io.Writer, vals ...uint64) (tw int, err error) { + buf := make([]byte, binary.MaxVarintLen64) + for _, val := range vals { + n := binary.PutUvarint(buf, val) + var nw int + nw, err = w.Write(buf[:n]) + tw += nw + if err != nil { + return tw, err + } + } + return tw, err +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go new file mode 100644 index 00000000..edb986d0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index.go @@ -0,0 +1,638 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "bytes" + "container/heap" + "encoding/binary" + "fmt" + "reflect" + "sort" + "sync" + "sync/atomic" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/scorch/segment" +) + +type asynchSegmentResult struct { + dictItr segment.DictionaryIterator + + index int + docs *roaring.Bitmap + + postings segment.PostingsList + + err error +} + +var reflectStaticSizeIndexSnapshot int + +func init() { + var is interface{} = IndexSnapshot{} + reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size()) +} + +type IndexSnapshot struct { + parent *Scorch + segment []*SegmentSnapshot + offsets []uint64 + internal map[string][]byte + epoch uint64 + size uint64 + creator string + + m sync.Mutex // Protects the fields that follow. + refs int64 + + m2 sync.Mutex // Protects the fields that follow. + fieldTFRs map[string][]*IndexSnapshotTermFieldReader // keyed by field, recycled TFR's + fieldDicts map[string][]segment.TermDictionary // keyed by field, recycled dicts +} + +func (i *IndexSnapshot) Segments() []*SegmentSnapshot { + return i.segment +} + +func (i *IndexSnapshot) Internal() map[string][]byte { + return i.internal +} + +func (i *IndexSnapshot) AddRef() { + i.m.Lock() + i.refs++ + i.m.Unlock() +} + +func (i *IndexSnapshot) DecRef() (err error) { + i.m.Lock() + i.refs-- + if i.refs == 0 { + for _, s := range i.segment { + if s != nil { + err2 := s.segment.DecRef() + if err == nil { + err = err2 + } + } + } + if i.parent != nil { + go i.parent.AddEligibleForRemoval(i.epoch) + } + } + i.m.Unlock() + return err +} + +func (i *IndexSnapshot) Close() error { + return i.DecRef() +} + +func (i *IndexSnapshot) Size() int { + return int(i.size) +} + +func (i *IndexSnapshot) updateSize() { + i.size += uint64(reflectStaticSizeIndexSnapshot) + for _, s := range i.segment { + i.size += uint64(s.Size()) + } +} + +func (i *IndexSnapshot) newIndexSnapshotFieldDict(field string, makeItr func(i segment.TermDictionary) segment.DictionaryIterator) (*IndexSnapshotFieldDict, error) { + + results := make(chan *asynchSegmentResult) + for index, segment := range i.segment { + go func(index int, segment *SegmentSnapshot) { + dict, err := segment.Dictionary(field) + if err != nil { + results <- &asynchSegmentResult{err: err} + } else { + results <- &asynchSegmentResult{dictItr: makeItr(dict)} + } + }(index, segment) + } + + var err error + rv := &IndexSnapshotFieldDict{ + snapshot: i, + cursors: make([]*segmentDictCursor, 0, len(i.segment)), + } + for count := 0; count < len(i.segment); count++ { + asr := <-results + if asr.err != nil && err == nil { + err = asr.err + } else { + next, err2 := asr.dictItr.Next() + if err2 != nil && err == nil { + err = err2 + } + if next != nil { + rv.cursors = append(rv.cursors, &segmentDictCursor{ + itr: asr.dictItr, + curr: *next, + }) + } + } + } + // after ensuring we've read all items on channel + if err != nil { + return nil, err + } + // prepare heap + heap.Init(rv) + + return rv, nil +} + +func (i *IndexSnapshot) FieldDict(field string) (index.FieldDict, error) { + return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { + return i.Iterator() + }) +} + +func (i *IndexSnapshot) FieldDictRange(field string, startTerm []byte, + endTerm []byte) (index.FieldDict, error) { + return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { + return i.RangeIterator(string(startTerm), string(endTerm)) + }) +} + +func (i *IndexSnapshot) FieldDictPrefix(field string, + termPrefix []byte) (index.FieldDict, error) { + return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { + return i.PrefixIterator(string(termPrefix)) + }) +} + +func (i *IndexSnapshot) FieldDictRegexp(field string, + termRegex []byte) (index.FieldDict, error) { + return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { + return i.RegexpIterator(string(termRegex)) + }) +} + +func (i *IndexSnapshot) FieldDictFuzzy(field string, + term []byte, fuzziness int) (index.FieldDict, error) { + return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { + return i.FuzzyIterator(string(term), fuzziness) + }) +} + +func (i *IndexSnapshot) FieldDictOnly(field string, + onlyTerms [][]byte, includeCount bool) (index.FieldDict, error) { + return i.newIndexSnapshotFieldDict(field, func(i segment.TermDictionary) segment.DictionaryIterator { + return i.OnlyIterator(onlyTerms, includeCount) + }) +} + +func (i *IndexSnapshot) DocIDReaderAll() (index.DocIDReader, error) { + results := make(chan *asynchSegmentResult) + for index, segment := range i.segment { + go func(index int, segment *SegmentSnapshot) { + results <- &asynchSegmentResult{ + index: index, + docs: segment.DocNumbersLive(), + } + }(index, segment) + } + + return i.newDocIDReader(results) +} + +func (i *IndexSnapshot) DocIDReaderOnly(ids []string) (index.DocIDReader, error) { + results := make(chan *asynchSegmentResult) + for index, segment := range i.segment { + go func(index int, segment *SegmentSnapshot) { + docs, err := segment.DocNumbers(ids) + if err != nil { + results <- &asynchSegmentResult{err: err} + } else { + results <- &asynchSegmentResult{ + index: index, + docs: docs, + } + } + }(index, segment) + } + + return i.newDocIDReader(results) +} + +func (i *IndexSnapshot) newDocIDReader(results chan *asynchSegmentResult) (index.DocIDReader, error) { + rv := &IndexSnapshotDocIDReader{ + snapshot: i, + iterators: make([]roaring.IntIterable, len(i.segment)), + } + var err error + for count := 0; count < len(i.segment); count++ { + asr := <-results + if asr.err != nil && err != nil { + err = asr.err + } else { + rv.iterators[asr.index] = asr.docs.Iterator() + } + } + + if err != nil { + return nil, err + } + + return rv, nil +} + +func (i *IndexSnapshot) Fields() ([]string, error) { + // FIXME not making this concurrent for now as it's not used in hot path + // of any searches at the moment (just a debug aid) + fieldsMap := map[string]struct{}{} + for _, segment := range i.segment { + fields := segment.Fields() + for _, field := range fields { + fieldsMap[field] = struct{}{} + } + } + rv := make([]string, 0, len(fieldsMap)) + for k := range fieldsMap { + rv = append(rv, k) + } + return rv, nil +} + +func (i *IndexSnapshot) GetInternal(key []byte) ([]byte, error) { + return i.internal[string(key)], nil +} + +func (i *IndexSnapshot) DocCount() (uint64, error) { + var rv uint64 + for _, segment := range i.segment { + rv += segment.Count() + } + return rv, nil +} + +func (i *IndexSnapshot) Document(id string) (rv *document.Document, err error) { + // FIXME could be done more efficiently directly, but reusing for simplicity + tfr, err := i.TermFieldReader([]byte(id), "_id", false, false, false) + if err != nil { + return nil, err + } + defer func() { + if cerr := tfr.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + next, err := tfr.Next(nil) + if err != nil { + return nil, err + } + + if next == nil { + // no such doc exists + return nil, nil + } + + docNum, err := docInternalToNumber(next.ID) + if err != nil { + return nil, err + } + segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum) + + rv = document.NewDocument(id) + err = i.segment[segmentIndex].VisitDocument(localDocNum, func(name string, typ byte, value []byte, pos []uint64) bool { + if name == "_id" { + return true + } + switch typ { + case 't': + rv.AddField(document.NewTextField(name, pos, value)) + case 'n': + rv.AddField(document.NewNumericFieldFromBytes(name, pos, value)) + case 'd': + rv.AddField(document.NewDateTimeFieldFromBytes(name, pos, value)) + case 'b': + rv.AddField(document.NewBooleanFieldFromBytes(name, pos, value)) + case 'g': + rv.AddField(document.NewGeoPointFieldFromBytes(name, pos, value)) + } + + return true + }) + if err != nil { + return nil, err + } + + return rv, nil +} + +func (i *IndexSnapshot) segmentIndexAndLocalDocNumFromGlobal(docNum uint64) (int, uint64) { + segmentIndex := sort.Search(len(i.offsets), + func(x int) bool { + return i.offsets[x] > docNum + }) - 1 + + localDocNum := docNum - i.offsets[segmentIndex] + return int(segmentIndex), localDocNum +} + +func (i *IndexSnapshot) ExternalID(id index.IndexInternalID) (string, error) { + docNum, err := docInternalToNumber(id) + if err != nil { + return "", err + } + segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum) + + v, err := i.segment[segmentIndex].DocID(localDocNum) + if err != nil { + return "", err + } + if v == nil { + return "", fmt.Errorf("document number %d not found", docNum) + } + + return string(v), nil +} + +func (i *IndexSnapshot) InternalID(id string) (rv index.IndexInternalID, err error) { + // FIXME could be done more efficiently directly, but reusing for simplicity + tfr, err := i.TermFieldReader([]byte(id), "_id", false, false, false) + if err != nil { + return nil, err + } + defer func() { + if cerr := tfr.Close(); err == nil && cerr != nil { + err = cerr + } + }() + + next, err := tfr.Next(nil) + if err != nil || next == nil { + return nil, err + } + + return next.ID, nil +} + +func (i *IndexSnapshot) TermFieldReader(term []byte, field string, includeFreq, + includeNorm, includeTermVectors bool) (tfr index.TermFieldReader, err error) { + rv, dicts := i.allocTermFieldReaderDicts(field) + + rv.term = term + rv.field = field + rv.snapshot = i + if rv.postings == nil { + rv.postings = make([]segment.PostingsList, len(i.segment)) + } + if rv.iterators == nil { + rv.iterators = make([]segment.PostingsIterator, len(i.segment)) + } + rv.segmentOffset = 0 + rv.includeFreq = includeFreq + rv.includeNorm = includeNorm + rv.includeTermVectors = includeTermVectors + rv.currPosting = nil + rv.currID = rv.currID[:0] + + if dicts == nil { + dicts = make([]segment.TermDictionary, len(i.segment)) + for i, segment := range i.segment { + dict, err := segment.Dictionary(field) + if err != nil { + return nil, err + } + dicts[i] = dict + } + } + rv.dicts = dicts + + for i := range i.segment { + pl, err := dicts[i].PostingsList(term, nil, rv.postings[i]) + if err != nil { + return nil, err + } + rv.postings[i] = pl + rv.iterators[i] = pl.Iterator(includeFreq, includeNorm, includeTermVectors, rv.iterators[i]) + } + atomic.AddUint64(&i.parent.stats.TotTermSearchersStarted, uint64(1)) + return rv, nil +} + +func (i *IndexSnapshot) allocTermFieldReaderDicts(field string) ( + tfr *IndexSnapshotTermFieldReader, dicts []segment.TermDictionary) { + i.m2.Lock() + if i.fieldDicts != nil { + dicts = i.fieldDicts[field] + } + if i.fieldTFRs != nil { + tfrs := i.fieldTFRs[field] + last := len(tfrs) - 1 + if last >= 0 { + rv := tfrs[last] + tfrs[last] = nil + i.fieldTFRs[field] = tfrs[:last] + i.m2.Unlock() + return rv, dicts + } + } + i.m2.Unlock() + return &IndexSnapshotTermFieldReader{}, dicts +} + +func (i *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) { + i.m2.Lock() + if i.fieldTFRs == nil { + i.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{} + } + i.fieldTFRs[tfr.field] = append(i.fieldTFRs[tfr.field], tfr) + if i.fieldDicts == nil { + i.fieldDicts = map[string][]segment.TermDictionary{} + } + i.fieldDicts[tfr.field] = tfr.dicts + i.m2.Unlock() +} + +func docNumberToBytes(buf []byte, in uint64) []byte { + if len(buf) != 8 { + if cap(buf) >= 8 { + buf = buf[0:8] + } else { + buf = make([]byte, 8) + } + } + binary.BigEndian.PutUint64(buf, in) + return buf +} + +func docInternalToNumber(in index.IndexInternalID) (uint64, error) { + if len(in) != 8 { + return 0, fmt.Errorf("wrong len for IndexInternalID: %q", in) + } + return binary.BigEndian.Uint64(in), nil +} + +func (i *IndexSnapshot) DocumentVisitFieldTerms(id index.IndexInternalID, + fields []string, visitor index.DocumentFieldTermVisitor) error { + _, err := i.documentVisitFieldTerms(id, fields, visitor, nil) + return err +} + +func (i *IndexSnapshot) documentVisitFieldTerms(id index.IndexInternalID, + fields []string, visitor index.DocumentFieldTermVisitor, dvs segment.DocVisitState) ( + segment.DocVisitState, error) { + + docNum, err := docInternalToNumber(id) + if err != nil { + return nil, err + } + segmentIndex, localDocNum := i.segmentIndexAndLocalDocNumFromGlobal(docNum) + if segmentIndex >= len(i.segment) { + return nil, nil + } + + ss := i.segment[segmentIndex] + + if zaps, ok := ss.segment.(segment.DocumentFieldTermVisitable); ok { + // get the list of doc value persisted fields + pFields, err := zaps.VisitableDocValueFields() + if err != nil { + return nil, err + } + // assort the fields for which terms look up have to + // be performed runtime + dvPendingFields := extractDvPendingFields(fields, pFields) + // all fields are doc value persisted + if len(dvPendingFields) == 0 { + return zaps.VisitDocumentFieldTerms(localDocNum, fields, visitor, dvs) + } + + // concurrently trigger the runtime doc value preparations for + // pending fields as well as the visit of the persisted doc values + errCh := make(chan error, 1) + + go func() { + defer close(errCh) + err := ss.cachedDocs.prepareFields(dvPendingFields, ss) + if err != nil { + errCh <- err + } + }() + + // visit the requested persisted dv while the cache preparation in progress + dvs, err = zaps.VisitDocumentFieldTerms(localDocNum, fields, visitor, dvs) + if err != nil { + return nil, err + } + + // err out if fieldCache preparation failed + err = <-errCh + if err != nil { + return nil, err + } + + visitDocumentFieldCacheTerms(localDocNum, dvPendingFields, ss, visitor) + return dvs, nil + } + + return dvs, prepareCacheVisitDocumentFieldTerms(localDocNum, fields, ss, visitor) +} + +func prepareCacheVisitDocumentFieldTerms(localDocNum uint64, fields []string, + ss *SegmentSnapshot, visitor index.DocumentFieldTermVisitor) error { + err := ss.cachedDocs.prepareFields(fields, ss) + if err != nil { + return err + } + + visitDocumentFieldCacheTerms(localDocNum, fields, ss, visitor) + return nil +} + +func visitDocumentFieldCacheTerms(localDocNum uint64, fields []string, + ss *SegmentSnapshot, visitor index.DocumentFieldTermVisitor) { + + for _, field := range fields { + if cachedFieldDocs, exists := ss.cachedDocs.cache[field]; exists { + if tlist, exists := cachedFieldDocs.docs[localDocNum]; exists { + for { + i := bytes.Index(tlist, TermSeparatorSplitSlice) + if i < 0 { + break + } + visitor(field, tlist[0:i]) + tlist = tlist[i+1:] + } + } + } + } + +} + +func extractDvPendingFields(requestedFields, persistedFields []string) []string { + removeMap := make(map[string]struct{}, len(persistedFields)) + for _, str := range persistedFields { + removeMap[str] = struct{}{} + } + + rv := make([]string, 0, len(requestedFields)) + for _, s := range requestedFields { + if _, ok := removeMap[s]; !ok { + rv = append(rv, s) + } + } + return rv +} + +func (i *IndexSnapshot) DocValueReader(fields []string) (index.DocValueReader, error) { + return &DocValueReader{i: i, fields: fields}, nil +} + +type DocValueReader struct { + i *IndexSnapshot + fields []string + dvs segment.DocVisitState +} + +func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, + visitor index.DocumentFieldTermVisitor) (err error) { + dvr.dvs, err = dvr.i.documentVisitFieldTerms(id, dvr.fields, visitor, dvr.dvs) + return err +} + +func (i *IndexSnapshot) DumpAll() chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} + +func (i *IndexSnapshot) DumpDoc(id string) chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} + +func (i *IndexSnapshot) DumpFields() chan interface{} { + rv := make(chan interface{}) + go func() { + close(rv) + }() + return rv +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go new file mode 100644 index 00000000..2d229ca0 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_dict.go @@ -0,0 +1,93 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "container/heap" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/scorch/segment" +) + +type segmentDictCursor struct { + itr segment.DictionaryIterator + curr index.DictEntry +} + +type IndexSnapshotFieldDict struct { + snapshot *IndexSnapshot + cursors []*segmentDictCursor + entry index.DictEntry +} + +func (i *IndexSnapshotFieldDict) Len() int { return len(i.cursors) } +func (i *IndexSnapshotFieldDict) Less(a, b int) bool { + return i.cursors[a].curr.Term < i.cursors[b].curr.Term +} +func (i *IndexSnapshotFieldDict) Swap(a, b int) { + i.cursors[a], i.cursors[b] = i.cursors[b], i.cursors[a] +} + +func (i *IndexSnapshotFieldDict) Push(x interface{}) { + i.cursors = append(i.cursors, x.(*segmentDictCursor)) +} + +func (i *IndexSnapshotFieldDict) Pop() interface{} { + n := len(i.cursors) + x := i.cursors[n-1] + i.cursors = i.cursors[0 : n-1] + return x +} + +func (i *IndexSnapshotFieldDict) Next() (*index.DictEntry, error) { + if len(i.cursors) <= 0 { + return nil, nil + } + i.entry = i.cursors[0].curr + next, err := i.cursors[0].itr.Next() + if err != nil { + return nil, err + } + if next == nil { + // at end of this cursor, remove it + heap.Pop(i) + } else { + // modified heap, fix it + i.cursors[0].curr = *next + heap.Fix(i, 0) + } + // look for any other entries with the exact same term + for len(i.cursors) > 0 && i.cursors[0].curr.Term == i.entry.Term { + i.entry.Count += i.cursors[0].curr.Count + next, err := i.cursors[0].itr.Next() + if err != nil { + return nil, err + } + if next == nil { + // at end of this cursor, remove it + heap.Pop(i) + } else { + // modified heap, fix it + i.cursors[0].curr = *next + heap.Fix(i, 0) + } + } + + return &i.entry, nil +} + +func (i *IndexSnapshotFieldDict) Close() error { + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go new file mode 100644 index 00000000..27da2086 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_doc.go @@ -0,0 +1,80 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "bytes" + "reflect" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/size" +) + +var reflectStaticSizeIndexSnapshotDocIDReader int + +func init() { + var isdr IndexSnapshotDocIDReader + reflectStaticSizeIndexSnapshotDocIDReader = int(reflect.TypeOf(isdr).Size()) +} + +type IndexSnapshotDocIDReader struct { + snapshot *IndexSnapshot + iterators []roaring.IntIterable + segmentOffset int +} + +func (i *IndexSnapshotDocIDReader) Size() int { + return reflectStaticSizeIndexSnapshotDocIDReader + size.SizeOfPtr +} + +func (i *IndexSnapshotDocIDReader) Next() (index.IndexInternalID, error) { + for i.segmentOffset < len(i.iterators) { + if !i.iterators[i.segmentOffset].HasNext() { + i.segmentOffset++ + continue + } + next := i.iterators[i.segmentOffset].Next() + // make segment number into global number by adding offset + globalOffset := i.snapshot.offsets[i.segmentOffset] + return docNumberToBytes(nil, uint64(next)+globalOffset), nil + } + return nil, nil +} + +func (i *IndexSnapshotDocIDReader) Advance(ID index.IndexInternalID) (index.IndexInternalID, error) { + // FIXME do something better + next, err := i.Next() + if err != nil { + return nil, err + } + if next == nil { + return nil, nil + } + for bytes.Compare(next, ID) < 0 { + next, err = i.Next() + if err != nil { + return nil, err + } + if next == nil { + break + } + } + return next, nil +} + +func (i *IndexSnapshotDocIDReader) Close() error { + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go new file mode 100644 index 00000000..89af3be4 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_index_tfr.go @@ -0,0 +1,185 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "bytes" + "fmt" + "reflect" + "sync/atomic" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/size" +) + +var reflectStaticSizeIndexSnapshotTermFieldReader int + +func init() { + var istfr IndexSnapshotTermFieldReader + reflectStaticSizeIndexSnapshotTermFieldReader = int(reflect.TypeOf(istfr).Size()) +} + +type IndexSnapshotTermFieldReader struct { + term []byte + field string + snapshot *IndexSnapshot + dicts []segment.TermDictionary + postings []segment.PostingsList + iterators []segment.PostingsIterator + segmentOffset int + includeFreq bool + includeNorm bool + includeTermVectors bool + currPosting segment.Posting + currID index.IndexInternalID +} + +func (i *IndexSnapshotTermFieldReader) Size() int { + sizeInBytes := reflectStaticSizeIndexSnapshotTermFieldReader + size.SizeOfPtr + + len(i.term) + + len(i.field) + + len(i.currID) + + for _, entry := range i.postings { + sizeInBytes += entry.Size() + } + + for _, entry := range i.iterators { + sizeInBytes += entry.Size() + } + + if i.currPosting != nil { + sizeInBytes += i.currPosting.Size() + } + + return sizeInBytes +} + +func (i *IndexSnapshotTermFieldReader) Next(preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { + rv := preAlloced + if rv == nil { + rv = &index.TermFieldDoc{} + } + // find the next hit + for i.segmentOffset < len(i.postings) { + next, err := i.iterators[i.segmentOffset].Next() + if err != nil { + return nil, err + } + if next != nil { + // make segment number into global number by adding offset + globalOffset := i.snapshot.offsets[i.segmentOffset] + nnum := next.Number() + rv.ID = docNumberToBytes(rv.ID, nnum+globalOffset) + i.postingToTermFieldDoc(next, rv) + + i.currID = rv.ID + i.currPosting = next + return rv, nil + } + i.segmentOffset++ + } + return nil, nil +} + +func (i *IndexSnapshotTermFieldReader) postingToTermFieldDoc(next segment.Posting, rv *index.TermFieldDoc) { + if i.includeFreq { + rv.Freq = next.Frequency() + } + if i.includeNorm { + rv.Norm = next.Norm() + } + if i.includeTermVectors { + locs := next.Locations() + if cap(rv.Vectors) < len(locs) { + rv.Vectors = make([]*index.TermFieldVector, len(locs)) + backing := make([]index.TermFieldVector, len(locs)) + for i := range backing { + rv.Vectors[i] = &backing[i] + } + } + rv.Vectors = rv.Vectors[:len(locs)] + for i, loc := range locs { + *rv.Vectors[i] = index.TermFieldVector{ + Start: loc.Start(), + End: loc.End(), + Pos: loc.Pos(), + ArrayPositions: loc.ArrayPositions(), + Field: loc.Field(), + } + } + } +} + +func (i *IndexSnapshotTermFieldReader) Advance(ID index.IndexInternalID, preAlloced *index.TermFieldDoc) (*index.TermFieldDoc, error) { + // FIXME do something better + // for now, if we need to seek backwards, then restart from the beginning + if i.currPosting != nil && bytes.Compare(i.currID, ID) >= 0 { + i2, err := i.snapshot.TermFieldReader(i.term, i.field, + i.includeFreq, i.includeNorm, i.includeTermVectors) + if err != nil { + return nil, err + } + *i = *(i2.(*IndexSnapshotTermFieldReader)) + } + num, err := docInternalToNumber(ID) + if err != nil { + return nil, fmt.Errorf("error converting to doc number % x - %v", ID, err) + } + segIndex, ldocNum := i.snapshot.segmentIndexAndLocalDocNumFromGlobal(num) + if segIndex >= len(i.snapshot.segment) { + return nil, fmt.Errorf("computed segment index %d out of bounds %d", + segIndex, len(i.snapshot.segment)) + } + // skip directly to the target segment + i.segmentOffset = segIndex + next, err := i.iterators[i.segmentOffset].Advance(ldocNum) + if err != nil { + return nil, err + } + if next == nil { + // we jumped directly to the segment that should have contained it + // but it wasn't there, so reuse Next() which should correctly + // get the next hit after it (we moved i.segmentOffset) + return i.Next(preAlloced) + } + + if preAlloced == nil { + preAlloced = &index.TermFieldDoc{} + } + preAlloced.ID = docNumberToBytes(preAlloced.ID, next.Number()+ + i.snapshot.offsets[segIndex]) + i.postingToTermFieldDoc(next, preAlloced) + i.currID = preAlloced.ID + i.currPosting = next + return preAlloced, nil +} + +func (i *IndexSnapshotTermFieldReader) Count() uint64 { + var rv uint64 + for _, posting := range i.postings { + rv += posting.Count() + } + return rv +} + +func (i *IndexSnapshotTermFieldReader) Close() error { + if i.snapshot != nil { + atomic.AddUint64(&i.snapshot.parent.stats.TotTermSearchersFinished, uint64(1)) + i.snapshot.recycleTermFieldReader(i) + } + return nil +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_rollback.go b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_rollback.go new file mode 100644 index 00000000..24700331 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_rollback.go @@ -0,0 +1,173 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "fmt" + "log" + + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/boltdb/bolt" +) + +type RollbackPoint struct { + epoch uint64 + meta map[string][]byte +} + +func (r *RollbackPoint) GetInternal(key []byte) []byte { + return r.meta[string(key)] +} + +// RollbackPoints returns an array of rollback points available for +// the application to rollback to, with more recent rollback points +// (higher epochs) coming first. +func (s *Scorch) RollbackPoints() ([]*RollbackPoint, error) { + if s.rootBolt == nil { + return nil, fmt.Errorf("RollbackPoints: root is nil") + } + + // start a read-only bolt transaction + tx, err := s.rootBolt.Begin(false) + if err != nil { + return nil, fmt.Errorf("RollbackPoints: failed to start" + + " read-only transaction") + } + + // read-only bolt transactions to be rolled back + defer func() { + _ = tx.Rollback() + }() + + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return nil, nil + } + + rollbackPoints := []*RollbackPoint{} + + c1 := snapshots.Cursor() + for k, _ := c1.Last(); k != nil; k, _ = c1.Prev() { + _, snapshotEpoch, err := segment.DecodeUvarintAscending(k) + if err != nil { + log.Printf("RollbackPoints:"+ + " unable to parse segment epoch %x, continuing", k) + continue + } + + snapshot := snapshots.Bucket(k) + if snapshot == nil { + log.Printf("RollbackPoints:"+ + " snapshot key, but bucket missing %x, continuing", k) + continue + } + + meta := map[string][]byte{} + c2 := snapshot.Cursor() + for j, _ := c2.First(); j != nil; j, _ = c2.Next() { + if j[0] == boltInternalKey[0] { + internalBucket := snapshot.Bucket(j) + err = internalBucket.ForEach(func(key []byte, val []byte) error { + copiedVal := append([]byte(nil), val...) + meta[string(key)] = copiedVal + return nil + }) + if err != nil { + break + } + } + } + + if err != nil { + log.Printf("RollbackPoints:"+ + " failed in fetching internal data: %v", err) + continue + } + + rollbackPoints = append(rollbackPoints, &RollbackPoint{ + epoch: snapshotEpoch, + meta: meta, + }) + } + + return rollbackPoints, nil +} + +// Rollback atomically and durably (if unsafeBatch is unset) brings +// the store back to the point in time as represented by the +// RollbackPoint. Rollback() should only be passed a RollbackPoint +// that came from the same store using the RollbackPoints() API. +func (s *Scorch) Rollback(to *RollbackPoint) error { + if to == nil { + return fmt.Errorf("Rollback: RollbackPoint is nil") + } + + if s.rootBolt == nil { + return fmt.Errorf("Rollback: root is nil") + } + + revert := &snapshotReversion{} + + s.rootLock.Lock() + + err := s.rootBolt.View(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(boltSnapshotsBucket) + if snapshots == nil { + return fmt.Errorf("Rollback: no snapshots available") + } + + pos := segment.EncodeUvarintAscending(nil, to.epoch) + + snapshot := snapshots.Bucket(pos) + if snapshot == nil { + return fmt.Errorf("Rollback: snapshot not found") + } + + indexSnapshot, err := s.loadSnapshot(snapshot) + if err != nil { + return fmt.Errorf("Rollback: unable to load snapshot: %v", err) + } + + // add segments referenced by loaded index snapshot to the + // ineligibleForRemoval map + for _, segSnap := range indexSnapshot.segment { + filename := zapFileName(segSnap.id) + s.ineligibleForRemoval[filename] = true + } + + revert.snapshot = indexSnapshot + revert.applied = make(chan error) + revert.persisted = make(chan error) + + return nil + }) + + s.rootLock.Unlock() + + if err != nil { + return err + } + + // introduce the reversion + s.revertToSnapshots <- revert + + // block until this snapshot is applied + err = <-revert.applied + if err != nil { + return fmt.Errorf("Rollback: failed with err: %v", err) + } + + return <-revert.persisted +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go new file mode 100644 index 00000000..4053244d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/snapshot_segment.go @@ -0,0 +1,272 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "sync" + "sync/atomic" + + "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index/scorch/segment" + "github.com/blevesearch/bleve/size" +) + +var TermSeparator byte = 0xff + +var TermSeparatorSplitSlice = []byte{TermSeparator} + +type SegmentDictionarySnapshot struct { + s *SegmentSnapshot + d segment.TermDictionary +} + +func (s *SegmentDictionarySnapshot) PostingsList(term []byte, except *roaring.Bitmap, + prealloc segment.PostingsList) (segment.PostingsList, error) { + // TODO: if except is non-nil, perhaps need to OR it with s.s.deleted? + return s.d.PostingsList(term, s.s.deleted, prealloc) +} + +func (s *SegmentDictionarySnapshot) Iterator() segment.DictionaryIterator { + return s.d.Iterator() +} + +func (s *SegmentDictionarySnapshot) PrefixIterator(prefix string) segment.DictionaryIterator { + return s.d.PrefixIterator(prefix) +} + +func (s *SegmentDictionarySnapshot) RangeIterator(start, end string) segment.DictionaryIterator { + return s.d.RangeIterator(start, end) +} + +func (s *SegmentDictionarySnapshot) RegexpIterator(regex string) segment.DictionaryIterator { + return s.d.RegexpIterator(regex) +} + +func (s *SegmentDictionarySnapshot) FuzzyIterator(term string, + fuzziness int) segment.DictionaryIterator { + return s.d.FuzzyIterator(term, fuzziness) +} + +func (s *SegmentDictionarySnapshot) OnlyIterator(onlyTerms [][]byte, + includeCount bool) segment.DictionaryIterator { + return s.d.OnlyIterator(onlyTerms, includeCount) +} + +type SegmentSnapshot struct { + id uint64 + segment segment.Segment + deleted *roaring.Bitmap + creator string + + cachedDocs *cachedDocs +} + +func (s *SegmentSnapshot) Segment() segment.Segment { + return s.segment +} + +func (s *SegmentSnapshot) Deleted() *roaring.Bitmap { + return s.deleted +} + +func (s *SegmentSnapshot) Id() uint64 { + return s.id +} + +func (s *SegmentSnapshot) FullSize() int64 { + return int64(s.segment.Count()) +} + +func (s SegmentSnapshot) LiveSize() int64 { + return int64(s.Count()) +} + +func (s *SegmentSnapshot) Close() error { + return s.segment.Close() +} + +func (s *SegmentSnapshot) VisitDocument(num uint64, visitor segment.DocumentFieldValueVisitor) error { + return s.segment.VisitDocument(num, visitor) +} + +func (s *SegmentSnapshot) DocID(num uint64) ([]byte, error) { + return s.segment.DocID(num) +} + +func (s *SegmentSnapshot) Count() uint64 { + + rv := s.segment.Count() + if s.deleted != nil { + rv -= s.deleted.GetCardinality() + } + return rv +} + +func (s *SegmentSnapshot) Dictionary(field string) (segment.TermDictionary, error) { + d, err := s.segment.Dictionary(field) + if err != nil { + return nil, err + } + return &SegmentDictionarySnapshot{ + s: s, + d: d, + }, nil +} + +func (s *SegmentSnapshot) DocNumbers(docIDs []string) (*roaring.Bitmap, error) { + rv, err := s.segment.DocNumbers(docIDs) + if err != nil { + return nil, err + } + if s.deleted != nil { + rv.AndNot(s.deleted) + } + return rv, nil +} + +// DocNumbersLive returns bitsit containing doc numbers for all live docs +func (s *SegmentSnapshot) DocNumbersLive() *roaring.Bitmap { + rv := roaring.NewBitmap() + rv.AddRange(0, s.segment.Count()) + if s.deleted != nil { + rv.AndNot(s.deleted) + } + return rv +} + +func (s *SegmentSnapshot) Fields() []string { + return s.segment.Fields() +} + +func (s *SegmentSnapshot) Size() (rv int) { + rv = s.segment.Size() + if s.deleted != nil { + rv += int(s.deleted.GetSizeInBytes()) + } + rv += s.cachedDocs.Size() + return +} + +type cachedFieldDocs struct { + readyCh chan struct{} // closed when the cachedFieldDocs.docs is ready to be used. + err error // Non-nil if there was an error when preparing this cachedFieldDocs. + docs map[uint64][]byte // Keyed by localDocNum, value is a list of terms delimited by 0xFF. + size uint64 +} + +func (cfd *cachedFieldDocs) prepareFields(field string, ss *SegmentSnapshot) { + defer close(cfd.readyCh) + + cfd.size += uint64(size.SizeOfUint64) /* size field */ + dict, err := ss.segment.Dictionary(field) + if err != nil { + cfd.err = err + return + } + + var postings segment.PostingsList + var postingsItr segment.PostingsIterator + + dictItr := dict.Iterator() + next, err := dictItr.Next() + for err == nil && next != nil { + var err1 error + postings, err1 = dict.PostingsList([]byte(next.Term), nil, postings) + if err1 != nil { + cfd.err = err1 + return + } + + cfd.size += uint64(size.SizeOfUint64) /* map key */ + postingsItr = postings.Iterator(false, false, false, postingsItr) + nextPosting, err2 := postingsItr.Next() + for err2 == nil && nextPosting != nil { + docNum := nextPosting.Number() + cfd.docs[docNum] = append(cfd.docs[docNum], []byte(next.Term)...) + cfd.docs[docNum] = append(cfd.docs[docNum], TermSeparator) + cfd.size += uint64(len(next.Term) + 1) // map value + nextPosting, err2 = postingsItr.Next() + } + + if err2 != nil { + cfd.err = err2 + return + } + + next, err = dictItr.Next() + } + + if err != nil { + cfd.err = err + return + } +} + +type cachedDocs struct { + m sync.Mutex // As the cache is asynchronously prepared, need a lock + cache map[string]*cachedFieldDocs // Keyed by field + size uint64 +} + +func (c *cachedDocs) prepareFields(wantedFields []string, ss *SegmentSnapshot) error { + c.m.Lock() + if c.cache == nil { + c.cache = make(map[string]*cachedFieldDocs, len(ss.Fields())) + } + + for _, field := range wantedFields { + _, exists := c.cache[field] + if !exists { + c.cache[field] = &cachedFieldDocs{ + readyCh: make(chan struct{}), + docs: make(map[uint64][]byte), + } + + go c.cache[field].prepareFields(field, ss) + } + } + + for _, field := range wantedFields { + cachedFieldDocs := c.cache[field] + c.m.Unlock() + <-cachedFieldDocs.readyCh + + if cachedFieldDocs.err != nil { + return cachedFieldDocs.err + } + c.m.Lock() + } + c.updateSizeLOCKED() + + c.m.Unlock() + return nil +} + +func (c *cachedDocs) Size() int { + return int(atomic.LoadUint64(&c.size)) +} + +func (c *cachedDocs) updateSizeLOCKED() { + sizeInBytes := 0 + for k, v := range c.cache { // cachedFieldDocs + sizeInBytes += len(k) + if v != nil { + for _, entry := range v.docs { // docs + sizeInBytes += 8 /* size of uint64 */ + len(entry) + } + } + } + atomic.StoreUint64(&c.size, uint64(sizeInBytes)) +} diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/stats.go b/vendor/github.com/blevesearch/bleve/index/scorch/stats.go new file mode 100644 index 00000000..e9bcd91d --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/index/scorch/stats.go @@ -0,0 +1,129 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorch + +import ( + "encoding/json" + "reflect" + "sync/atomic" +) + +// Stats tracks statistics about the index, fields that are +// prefixed like CurXxxx are gauges (can go up and down), +// and fields that are prefixed like TotXxxx are monotonically +// increasing counters. +type Stats struct { + TotUpdates uint64 + TotDeletes uint64 + + TotBatches uint64 + TotBatchesEmpty uint64 + TotBatchIntroTime uint64 + MaxBatchIntroTime uint64 + + TotOnErrors uint64 + + TotAnalysisTime uint64 + TotIndexTime uint64 + + TotIndexedPlainTextBytes uint64 + + TotTermSearchersStarted uint64 + TotTermSearchersFinished uint64 + + TotIntroduceLoop uint64 + TotIntroduceSegmentBeg uint64 + TotIntroduceSegmentEnd uint64 + TotIntroducePersistBeg uint64 + TotIntroducePersistEnd uint64 + TotIntroduceMergeBeg uint64 + TotIntroduceMergeEnd uint64 + TotIntroduceRevertBeg uint64 + TotIntroduceRevertEnd uint64 + + TotIntroducedItems uint64 + TotIntroducedSegmentsBatch uint64 + TotIntroducedSegmentsMerge uint64 + + TotPersistLoopBeg uint64 + TotPersistLoopErr uint64 + TotPersistLoopProgress uint64 + TotPersistLoopWait uint64 + TotPersistLoopWaitNotified uint64 + TotPersistLoopEnd uint64 + + TotPersistedItems uint64 + TotPersistedSegments uint64 + + TotPersisterSlowMergerPause uint64 + TotPersisterSlowMergerResume uint64 + + TotFileMergeLoopBeg uint64 + TotFileMergeLoopErr uint64 + TotFileMergeLoopEnd uint64 + + TotFileMergePlan uint64 + TotFileMergePlanErr uint64 + TotFileMergePlanNone uint64 + TotFileMergePlanOk uint64 + + TotFileMergePlanTasks uint64 + TotFileMergePlanTasksDone uint64 + TotFileMergePlanTasksErr uint64 + TotFileMergePlanTasksSegments uint64 + TotFileMergePlanTasksSegmentsEmpty uint64 + + TotFileMergeSegmentsEmpty uint64 + TotFileMergeSegments uint64 + TotFileMergeWrittenBytes uint64 + + TotFileMergeZapBeg uint64 + TotFileMergeZapEnd uint64 + TotFileMergeZapTime uint64 + MaxFileMergeZapTime uint64 + + TotFileMergeIntroductions uint64 + TotFileMergeIntroductionsDone uint64 + + TotMemMergeBeg uint64 + TotMemMergeErr uint64 + TotMemMergeDone uint64 + TotMemMergeZapBeg uint64 + TotMemMergeZapEnd uint64 + TotMemMergeZapTime uint64 + MaxMemMergeZapTime uint64 + TotMemMergeSegments uint64 +} + +// atomically populates the returned map +func (s *Stats) ToMap() map[string]interface{} { + m := map[string]interface{}{} + sve := reflect.ValueOf(s).Elem() + svet := sve.Type() + for i := 0; i < svet.NumField(); i++ { + svef := sve.Field(i) + if svef.CanAddr() { + svefp := svef.Addr().Interface() + m[svet.Field(i).Name] = atomic.LoadUint64(svefp.(*uint64)) + } + } + return m +} + +// MarshalJSON implements json.Marshaler, and in contrast to standard +// json marshaling provides atomic safety +func (s *Stats) MarshalJSON() ([]byte, error) { + return json.Marshal(s.ToMap()) +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go index 77d523c3..ea7243ea 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/index_reader.go @@ -15,11 +15,20 @@ package upsidedown import ( + "reflect" + "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" ) +var reflectStaticSizeIndexReader int + +func init() { + var ir IndexReader + reflectStaticSizeIndexReader = int(reflect.TypeOf(ir).Size()) +} + type IndexReader struct { index *UpsideDownCouch kvreader store.KVReader @@ -201,3 +210,17 @@ func incrementBytes(in []byte) []byte { } return rv } + +func (i *IndexReader) DocValueReader(fields []string) (index.DocValueReader, error) { + return &DocValueReader{i: i, fields: fields}, nil +} + +type DocValueReader struct { + i *IndexReader + fields []string +} + +func (dvr *DocValueReader) VisitDocValues(id index.IndexInternalID, + visitor index.DocumentFieldTermVisitor) error { + return dvr.i.DocumentVisitFieldTerms(id, dvr.fields, visitor) +} diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go index 76032bf5..bc0fef11 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/reader.go @@ -16,13 +16,27 @@ package upsidedown import ( "bytes" + "reflect" "sort" "sync/atomic" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeUpsideDownCouchTermFieldReader int +var reflectStaticSizeUpsideDownCouchDocIDReader int + +func init() { + var tfr UpsideDownCouchTermFieldReader + reflectStaticSizeUpsideDownCouchTermFieldReader = + int(reflect.TypeOf(tfr).Size()) + var cdr UpsideDownCouchDocIDReader + reflectStaticSizeUpsideDownCouchDocIDReader = + int(reflect.TypeOf(cdr).Size()) +} + type UpsideDownCouchTermFieldReader struct { count uint64 indexReader *IndexReader @@ -35,6 +49,19 @@ type UpsideDownCouchTermFieldReader struct { includeTermVectors bool } +func (r *UpsideDownCouchTermFieldReader) Size() int { + sizeInBytes := reflectStaticSizeUpsideDownCouchTermFieldReader + size.SizeOfPtr + + len(r.term) + + r.tfrPrealloc.Size() + + len(r.keyBuf) + + if r.tfrNext != nil { + sizeInBytes += r.tfrNext.Size() + } + + return sizeInBytes +} + func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16, includeFreq, includeNorm, includeTermVectors bool) (*UpsideDownCouchTermFieldReader, error) { bufNeeded := termFrequencyRowKeySize(term, nil) if bufNeeded < dictionaryRowKeySize(term) { @@ -174,8 +201,18 @@ type UpsideDownCouchDocIDReader struct { onlyMode bool } -func newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDocIDReader, error) { +func (r *UpsideDownCouchDocIDReader) Size() int { + sizeInBytes := reflectStaticSizeUpsideDownCouchDocIDReader + + reflectStaticSizeIndexReader + size.SizeOfPtr + for _, entry := range r.only { + sizeInBytes += size.SizeOfString + len(entry) + } + + return sizeInBytes +} + +func newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDocIDReader, error) { startBytes := []byte{0x0} endBytes := []byte{0xff} @@ -190,15 +227,18 @@ func newUpsideDownCouchDocIDReader(indexReader *IndexReader) (*UpsideDownCouchDo } func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) (*UpsideDownCouchDocIDReader, error) { + // we don't actually own the list of ids, so if before we sort we must copy + idsCopy := make([]string, len(ids)) + copy(idsCopy, ids) // ensure ids are sorted - sort.Strings(ids) + sort.Strings(idsCopy) startBytes := []byte{0x0} - if len(ids) > 0 { - startBytes = []byte(ids[0]) + if len(idsCopy) > 0 { + startBytes = []byte(idsCopy[0]) } endBytes := []byte{0xff} - if len(ids) > 0 { - endBytes = incrementBytes([]byte(ids[len(ids)-1])) + if len(idsCopy) > 0 { + endBytes = incrementBytes([]byte(idsCopy[len(idsCopy)-1])) } bisr := NewBackIndexRow(startBytes, nil, nil) bier := NewBackIndexRow(endBytes, nil, nil) @@ -207,7 +247,7 @@ func newUpsideDownCouchDocIDReaderOnly(indexReader *IndexReader, ids []string) ( return &UpsideDownCouchDocIDReader{ indexReader: indexReader, iterator: it, - only: ids, + only: idsCopy, onlyMode: true, }, nil } diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go index 7e503ae0..ba50314c 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/row.go @@ -20,10 +20,22 @@ import ( "fmt" "io" "math" + "reflect" + "github.com/blevesearch/bleve/size" "github.com/golang/protobuf/proto" ) +var reflectStaticSizeTermFrequencyRow int +var reflectStaticSizeTermVector int + +func init() { + var tfr TermFrequencyRow + reflectStaticSizeTermFrequencyRow = int(reflect.TypeOf(tfr).Size()) + var tv TermVector + reflectStaticSizeTermVector = int(reflect.TypeOf(tv).Size()) +} + const ByteSeparator byte = 0xff type UpsideDownCouchRowStream chan UpsideDownCouchRow @@ -358,6 +370,11 @@ type TermVector struct { end uint64 } +func (tv *TermVector) Size() int { + return reflectStaticSizeTermVector + size.SizeOfPtr + + len(tv.arrayPositions)*size.SizeOfUint64 +} + func (tv *TermVector) String() string { return fmt.Sprintf("Field: %d Pos: %d Start: %d End %d ArrayPositions: %#v", tv.field, tv.pos, tv.start, tv.end, tv.arrayPositions) } @@ -371,6 +388,18 @@ type TermFrequencyRow struct { field uint16 } +func (tfr *TermFrequencyRow) Size() int { + sizeInBytes := reflectStaticSizeTermFrequencyRow + + len(tfr.term) + + len(tfr.doc) + + for _, entry := range tfr.vectors { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} + func (tfr *TermFrequencyRow) Term() []byte { return tfr.term } diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go index a8ef538e..70e6e457 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go @@ -293,7 +293,7 @@ func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRowsAll [][]Upsi } func (udc *UpsideDownCouch) Open() (err error) { - //acquire the write mutex for the duratin of Open() + // acquire the write mutex for the duration of Open() udc.writeMutex.Lock() defer udc.writeMutex.Unlock() @@ -837,6 +837,11 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { docBackIndexRowErr = err return } + defer func() { + if cerr := kvreader.Close(); err == nil && cerr != nil { + docBackIndexRowErr = cerr + } + }() for docID, doc := range batch.IndexOps { backIndexRow, err := backIndexRowForDoc(kvreader, index.IndexInternalID(docID)) @@ -847,12 +852,6 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { docBackIndexRowCh <- &docBackIndexRow{docID, doc, backIndexRow} } - - err = kvreader.Close() - if err != nil { - docBackIndexRowErr = err - return - } }() // wait for analysis result diff --git a/vendor/github.com/blevesearch/bleve/index_alias_impl.go b/vendor/github.com/blevesearch/bleve/index_alias_impl.go index 9e9a3594..f678a059 100644 --- a/vendor/github.com/blevesearch/bleve/index_alias_impl.go +++ b/vendor/github.com/blevesearch/bleve/index_alias_impl.go @@ -15,12 +15,11 @@ package bleve import ( + "context" "sort" "sync" "time" - "golang.org/x/net/context" - "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" diff --git a/vendor/github.com/blevesearch/bleve/index_impl.go b/vendor/github.com/blevesearch/bleve/index_impl.go index c1c4608c..c969f375 100644 --- a/vendor/github.com/blevesearch/bleve/index_impl.go +++ b/vendor/github.com/blevesearch/bleve/index_impl.go @@ -15,6 +15,7 @@ package bleve import ( + "context" "encoding/json" "fmt" "os" @@ -22,8 +23,6 @@ import ( "sync/atomic" "time" - "golang.org/x/net/context" - "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" @@ -51,6 +50,12 @@ const storePath = "store" var mappingInternalKey = []byte("_mapping") +const SearchQueryStartCallbackKey = "_search_query_start_callback_key" +const SearchQueryEndCallbackKey = "_search_query_end_callback_key" + +type SearchQueryStartCallbackFn func(size uint64) error +type SearchQueryEndCallbackFn func(size uint64) error + func indexStorePath(path string) string { return path + string(os.PathSeparator) + storePath } @@ -253,6 +258,24 @@ func (i *indexImpl) Index(id string, data interface{}) (err error) { return } +// IndexAdvanced takes a document.Document object +// skips the mapping and indexes it. +func (i *indexImpl) IndexAdvanced(doc *document.Document) (err error) { + if doc.ID == "" { + return ErrorEmptyID + } + + i.mutex.RLock() + defer i.mutex.RUnlock() + + if !i.open { + return ErrorIndexClosed + } + + err = i.i.Update(doc) + return +} + // Delete entries for the specified identifier from // the index. func (i *indexImpl) Delete(id string) (err error) { @@ -345,8 +368,70 @@ func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) { return i.SearchInContext(context.Background(), req) } +var documentMatchEmptySize int +var searchContextEmptySize int +var facetResultEmptySize int +var documentEmptySize int + +func init() { + var dm search.DocumentMatch + documentMatchEmptySize = dm.Size() + + var sc search.SearchContext + searchContextEmptySize = sc.Size() + + var fr search.FacetResult + facetResultEmptySize = fr.Size() + + var d document.Document + documentEmptySize = d.Size() +} + +// memNeededForSearch is a helper function that returns an estimate of RAM +// needed to execute a search request. +func memNeededForSearch(req *SearchRequest, + searcher search.Searcher, + topnCollector *collector.TopNCollector) uint64 { + + backingSize := req.Size + req.From + 1 + if req.Size+req.From > collector.PreAllocSizeSkipCap { + backingSize = collector.PreAllocSizeSkipCap + 1 + } + numDocMatches := backingSize + searcher.DocumentMatchPoolSize() + + estimate := 0 + + // overhead, size in bytes from collector + estimate += topnCollector.Size() + + // pre-allocing DocumentMatchPool + estimate += searchContextEmptySize + numDocMatches*documentMatchEmptySize + + // searcher overhead + estimate += searcher.Size() + + // overhead from results, lowestMatchOutsideResults + estimate += (numDocMatches + 1) * documentMatchEmptySize + + // additional overhead from SearchResult + estimate += reflectStaticSizeSearchResult + reflectStaticSizeSearchStatus + + // overhead from facet results + if req.Facets != nil { + estimate += len(req.Facets) * facetResultEmptySize + } + + // highlighting, store + if len(req.Fields) > 0 || req.Highlight != nil { + // Size + From => number of hits + estimate += (req.Size + req.From) * documentEmptySize + } + + return uint64(estimate) +} + // SearchInContext executes a search request operation within the provided -// Context. Returns a SearchResult object or an error. +// Context. Returns a SearchResult object or an error. func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr *SearchResult, err error) { i.mutex.RLock() defer i.mutex.RUnlock() @@ -411,6 +496,24 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr collector.SetFacetsBuilder(facetsBuilder) } + memNeeded := memNeededForSearch(req, searcher, collector) + if cb := ctx.Value(SearchQueryStartCallbackKey); cb != nil { + if cbF, ok := cb.(SearchQueryStartCallbackFn); ok { + err = cbF(memNeeded) + } + } + if err != nil { + return nil, err + } + + if cb := ctx.Value(SearchQueryEndCallbackKey); cb != nil { + if cbF, ok := cb.(SearchQueryEndCallbackFn); ok { + defer func() { + _ = cbF(memNeeded) + }() + } + } + err = collector.Collect(ctx, searcher, indexReader) if err != nil { return nil, err @@ -442,7 +545,8 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr doc, err := indexReader.Document(hit.ID) if err == nil && doc != nil { if len(req.Fields) > 0 { - for _, f := range req.Fields { + fieldsToLoad := deDuplicate(req.Fields) + for _, f := range fieldsToLoad { for _, docF := range doc.Fields { if f == "*" || docF.Name() == f { var value interface{} @@ -516,9 +620,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr return &SearchResult{ Status: &SearchStatus{ Total: 1, - Failed: 0, Successful: 1, - Errors: make(map[string]error), }, Request: req, Hits: hits, @@ -738,3 +840,16 @@ func (f *indexImplFieldDict) Close() error { } return f.indexReader.Close() } + +// helper function to remove duplicate entries from slice of strings +func deDuplicate(fields []string) []string { + entries := make(map[string]struct{}) + ret := []string{} + for _, entry := range fields { + if _, exists := entries[entry]; !exists { + entries[entry] = struct{}{} + ret = append(ret, entry) + } + } + return ret +} diff --git a/vendor/github.com/blevesearch/bleve/mapping/document.go b/vendor/github.com/blevesearch/bleve/mapping/document.go index 9bdb8596..6ec0c66b 100644 --- a/vendor/github.com/blevesearch/bleve/mapping/document.go +++ b/vendor/github.com/blevesearch/bleve/mapping/document.go @@ -15,6 +15,7 @@ package mapping import ( + "encoding" "encoding/json" "fmt" "reflect" @@ -178,6 +179,7 @@ OUTER: continue OUTER } } + break } return current } @@ -481,6 +483,17 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, fieldMapping := newDateTimeFieldMappingDynamic(context.im) fieldMapping.processTime(property, pathString, path, indexes, context) } + case encoding.TextMarshaler: + txt, err := property.MarshalText() + if err == nil && subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type == "text" { + fieldMapping.processString(string(txt), pathString, path, indexes, context) + } + } + } + dm.walkDocument(property, path, indexes, context) default: if subDocMapping != nil { for _, fieldMapping := range subDocMapping.Fields { @@ -491,7 +504,7 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, } dm.walkDocument(property, path, indexes, context) } - case reflect.Map: + case reflect.Map, reflect.Slice: if subDocMapping != nil { for _, fieldMapping := range subDocMapping.Fields { if fieldMapping.Type == "geopoint" { @@ -500,6 +513,27 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, } } dm.walkDocument(property, path, indexes, context) + case reflect.Ptr: + if !propertyValue.IsNil() { + switch property := property.(type) { + case encoding.TextMarshaler: + + txt, err := property.MarshalText() + if err == nil && subDocMapping != nil { + // index by explicit mapping + for _, fieldMapping := range subDocMapping.Fields { + if fieldMapping.Type == "text" { + fieldMapping.processString(string(txt), pathString, path, indexes, context) + } + } + } else { + dm.walkDocument(property, path, indexes, context) + } + + default: + dm.walkDocument(property, path, indexes, context) + } + } default: dm.walkDocument(property, path, indexes, context) } diff --git a/vendor/github.com/blevesearch/bleve/mapping/field.go b/vendor/github.com/blevesearch/bleve/mapping/field.go index 9f1928ca..278faa1a 100644 --- a/vendor/github.com/blevesearch/bleve/mapping/field.go +++ b/vendor/github.com/blevesearch/bleve/mapping/field.go @@ -26,8 +26,9 @@ import ( // control the default behavior for dynamic fields (those not explicitly mapped) var ( - IndexDynamic = true - StoreDynamic = true + IndexDynamic = true + StoreDynamic = true + DocValuesDynamic = true // TODO revisit default? ) // A FieldMapping describes how a specific item @@ -54,6 +55,10 @@ type FieldMapping struct { IncludeTermVectors bool `json:"include_term_vectors,omitempty"` IncludeInAll bool `json:"include_in_all,omitempty"` DateFormat string `json:"date_format,omitempty"` + + // DocValues, if true makes the index uninverting possible for this field + // It is useful for faceting and sorting queries. + DocValues bool `json:"docvalues,omitempty"` } // NewTextFieldMapping returns a default field mapping for text @@ -64,6 +69,7 @@ func NewTextFieldMapping() *FieldMapping { Index: true, IncludeTermVectors: true, IncludeInAll: true, + DocValues: true, } } @@ -71,6 +77,7 @@ func newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { rv := NewTextFieldMapping() rv.Store = im.StoreDynamic rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic return rv } @@ -81,6 +88,7 @@ func NewNumericFieldMapping() *FieldMapping { Store: true, Index: true, IncludeInAll: true, + DocValues: true, } } @@ -88,6 +96,7 @@ func newNumericFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { rv := NewNumericFieldMapping() rv.Store = im.StoreDynamic rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic return rv } @@ -98,6 +107,7 @@ func NewDateTimeFieldMapping() *FieldMapping { Store: true, Index: true, IncludeInAll: true, + DocValues: true, } } @@ -105,6 +115,7 @@ func newDateTimeFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { rv := NewDateTimeFieldMapping() rv.Store = im.StoreDynamic rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic return rv } @@ -115,6 +126,7 @@ func NewBooleanFieldMapping() *FieldMapping { Store: true, Index: true, IncludeInAll: true, + DocValues: true, } } @@ -122,6 +134,7 @@ func newBooleanFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { rv := NewBooleanFieldMapping() rv.Store = im.StoreDynamic rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic return rv } @@ -132,6 +145,7 @@ func NewGeoPointFieldMapping() *FieldMapping { Store: true, Index: true, IncludeInAll: true, + DocValues: true, } } @@ -147,6 +161,9 @@ func (fm *FieldMapping) Options() document.IndexingOptions { if fm.IncludeTermVectors { rv |= document.IncludeTermVectors } + if fm.DocValues { + rv |= document.DocValues + } return rv } @@ -308,6 +325,11 @@ func (fm *FieldMapping) UnmarshalJSON(data []byte) error { if err != nil { return err } + case "docvalues": + err := json.Unmarshal(v, &fm.DocValues) + if err != nil { + return err + } default: invalidKeys = append(invalidKeys, k) } diff --git a/vendor/github.com/blevesearch/bleve/mapping/index.go b/vendor/github.com/blevesearch/bleve/mapping/index.go index 736e030b..fc5d12a7 100644 --- a/vendor/github.com/blevesearch/bleve/mapping/index.go +++ b/vendor/github.com/blevesearch/bleve/mapping/index.go @@ -50,6 +50,7 @@ type IndexMappingImpl struct { DefaultField string `json:"default_field"` StoreDynamic bool `json:"store_dynamic"` IndexDynamic bool `json:"index_dynamic"` + DocValuesDynamic bool `json:"docvalues_dynamic,omitempty"` CustomAnalysis *customAnalysis `json:"analysis,omitempty"` cache *registry.Cache } @@ -154,6 +155,7 @@ func NewIndexMapping() *IndexMappingImpl { DefaultField: defaultField, IndexDynamic: IndexDynamic, StoreDynamic: StoreDynamic, + DocValuesDynamic: DocValuesDynamic, CustomAnalysis: newCustomAnalysis(), cache: registry.NewCache(), } @@ -217,6 +219,7 @@ func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { im.TypeMapping = make(map[string]*DocumentMapping) im.StoreDynamic = StoreDynamic im.IndexDynamic = IndexDynamic + im.DocValuesDynamic = DocValuesDynamic var invalidKeys []string for k, v := range tmp { @@ -271,6 +274,11 @@ func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { if err != nil { return err } + case "docvalues_dynamic": + err := json.Unmarshal(v, &im.DocValuesDynamic) + if err != nil { + return err + } default: invalidKeys = append(invalidKeys, k) } @@ -289,7 +297,12 @@ func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error { } func (im *IndexMappingImpl) determineType(data interface{}) string { - // first see if the object implements Classifier + // first see if the object implements bleveClassifier + bleveClassifier, ok := data.(bleveClassifier) + if ok { + return bleveClassifier.BleveType() + } + // next see if the object implements Classifier classifier, ok := data.(Classifier) if ok { return classifier.Type() @@ -313,7 +326,7 @@ func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{} // see if the _all field was disabled allMapping := docMapping.documentMappingForPath("_all") - if allMapping == nil || (allMapping.Enabled != false) { + if allMapping == nil || allMapping.Enabled { field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors) doc.AddField(field) } @@ -334,7 +347,7 @@ func (im *IndexMappingImpl) newWalkContext(doc *document.Document, dm *DocumentM doc: doc, im: im, dm: dm, - excludedFromAll: []string{}, + excludedFromAll: []string{"_id"}, } } diff --git a/vendor/github.com/blevesearch/bleve/mapping/mapping.go b/vendor/github.com/blevesearch/bleve/mapping/mapping.go index 61002110..4a472811 100644 --- a/vendor/github.com/blevesearch/bleve/mapping/mapping.go +++ b/vendor/github.com/blevesearch/bleve/mapping/mapping.go @@ -22,12 +22,21 @@ import ( "github.com/blevesearch/bleve/document" ) -// A Classifier is an interface describing any object -// which knows how to identify its own type. +// A Classifier is an interface describing any object which knows how to +// identify its own type. Alternatively, if a struct already has a Type +// field or method in conflict, one can use BleveType instead. type Classifier interface { Type() string } +// A bleveClassifier is an interface describing any object which knows how +// to identify its own type. This is introduced as an alternative to the +// Classifier interface which often has naming conflicts with existing +// structures. +type bleveClassifier interface { + BleveType() string +} + var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags) // SetLog sets the logger used for logging diff --git a/vendor/github.com/blevesearch/bleve/query.go b/vendor/github.com/blevesearch/bleve/query.go index 1fecfa25..523db5ec 100644 --- a/vendor/github.com/blevesearch/bleve/query.go +++ b/vendor/github.com/blevesearch/bleve/query.go @@ -209,8 +209,8 @@ func NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightL return query.NewGeoBoundingBoxQuery(topLeftLon, topLeftLat, bottomRightLon, bottomRightLat) } -// NewGeoDistanceQuery creates a new Query for performing geo bounding -// box searches. The arguments describe a position and a distance. Documents +// NewGeoDistanceQuery creates a new Query for performing geo distance +// searches. The arguments describe a position and a distance. Documents // which have an indexed geo point which is less than or equal to the provided // distance from the given position will be returned. func NewGeoDistanceQuery(lon, lat float64, distance string) *query.GeoDistanceQuery { diff --git a/vendor/github.com/blevesearch/bleve/search.go b/vendor/github.com/blevesearch/bleve/search.go index 98790c9e..86ea4193 100644 --- a/vendor/github.com/blevesearch/bleve/search.go +++ b/vendor/github.com/blevesearch/bleve/search.go @@ -17,15 +17,29 @@ package bleve import ( "encoding/json" "fmt" + "reflect" "time" "github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/analysis/datetime/optional" + "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/collector" "github.com/blevesearch/bleve/search/query" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeSearchResult int +var reflectStaticSizeSearchStatus int + +func init() { + var sr SearchResult + reflectStaticSizeSearchResult = int(reflect.TypeOf(sr).Size()) + var ss SearchStatus + reflectStaticSizeSearchStatus = int(reflect.TypeOf(ss).Size()) +} + var cache = registry.NewCache() const defaultDateTimeParser = optional.Name @@ -170,6 +184,16 @@ func (fr *FacetRequest) AddDateTimeRange(name string, start, end time.Time) { fr.DateTimeRanges = append(fr.DateTimeRanges, &dateTimeRange{Name: name, Start: start, End: end}) } +// AddDateTimeRangeString adds a bucket to a field +// containing date values. +func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) { + if fr.DateTimeRanges == nil { + fr.DateTimeRanges = make([]*dateTimeRange, 0, 1) + } + fr.DateTimeRanges = append(fr.DateTimeRanges, + &dateTimeRange{Name: name, startString: start, endString: end}) +} + // AddNumericRange adds a bucket to a field // containing numeric values. Documents with a // numeric value falling into this range are @@ -422,6 +446,24 @@ type SearchResult struct { Facets search.FacetResults `json:"facets"` } +func (sr *SearchResult) Size() int { + sizeInBytes := reflectStaticSizeSearchResult + size.SizeOfPtr + + reflectStaticSizeSearchStatus + + for _, entry := range sr.Hits { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + for k, v := range sr.Facets { + sizeInBytes += size.SizeOfString + len(k) + + v.Size() + } + + return sizeInBytes +} + func (sr *SearchResult) String() string { rv := "" if sr.Total > 0 { @@ -471,5 +513,51 @@ func (sr *SearchResult) Merge(other *SearchResult) { if other.MaxScore > sr.MaxScore { sr.MaxScore = other.MaxScore } + if sr.Facets == nil && len(other.Facets) != 0 { + sr.Facets = other.Facets + return + } + sr.Facets.Merge(other.Facets) } + +// MemoryNeededForSearchResult is an exported helper function to determine the RAM +// needed to accommodate the results for a given search request. +func MemoryNeededForSearchResult(req *SearchRequest) uint64 { + if req == nil { + return 0 + } + + numDocMatches := req.Size + req.From + if req.Size+req.From > collector.PreAllocSizeSkipCap { + numDocMatches = collector.PreAllocSizeSkipCap + } + + estimate := 0 + + // overhead from the SearchResult structure + var sr SearchResult + estimate += sr.Size() + + var dm search.DocumentMatch + sizeOfDocumentMatch := dm.Size() + + // overhead from results + estimate += numDocMatches * sizeOfDocumentMatch + + // overhead from facet results + if req.Facets != nil { + var fr search.FacetResult + estimate += len(req.Facets) * fr.Size() + } + + // highlighting, store + var d document.Document + if len(req.Fields) > 0 || req.Highlight != nil { + for i := 0; i < (req.Size + req.From); i++ { + estimate += (req.Size + req.From) * d.Size() + } + } + + return uint64(estimate) +} diff --git a/vendor/github.com/blevesearch/bleve/search/collector.go b/vendor/github.com/blevesearch/bleve/search/collector.go index cba4829d..0d163a9d 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector.go +++ b/vendor/github.com/blevesearch/bleve/search/collector.go @@ -15,11 +15,10 @@ package search import ( + "context" "time" "github.com/blevesearch/bleve/index" - - "golang.org/x/net/context" ) type Collector interface { diff --git a/vendor/github.com/blevesearch/bleve/search/collector/heap.go b/vendor/github.com/blevesearch/bleve/search/collector/heap.go index 19f3a059..bdf72ead 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/heap.go +++ b/vendor/github.com/blevesearch/bleve/search/collector/heap.go @@ -34,11 +34,20 @@ func newStoreHeap(cap int, compare collectorCompare) *collectStoreHeap { return rv } -func (c *collectStoreHeap) Add(doc *search.DocumentMatch) { +func (c *collectStoreHeap) AddNotExceedingSize(doc *search.DocumentMatch, + size int) *search.DocumentMatch { + c.add(doc) + if c.Len() > size { + return c.removeLast() + } + return nil +} + +func (c *collectStoreHeap) add(doc *search.DocumentMatch) { heap.Push(c, doc) } -func (c *collectStoreHeap) RemoveLast() *search.DocumentMatch { +func (c *collectStoreHeap) removeLast() *search.DocumentMatch { return heap.Pop(c).(*search.DocumentMatch) } @@ -49,17 +58,12 @@ func (c *collectStoreHeap) Final(skip int, fixup collectorFixup) (search.Documen return make(search.DocumentMatchCollection, 0), nil } rv := make(search.DocumentMatchCollection, size) - for count > 0 { - count-- - - if count >= skip { - size-- - doc := heap.Pop(c).(*search.DocumentMatch) - rv[size] = doc - err := fixup(doc) - if err != nil { - return nil, err - } + for i := size - 1; i >= 0; i-- { + doc := heap.Pop(c).(*search.DocumentMatch) + rv[i] = doc + err := fixup(doc) + if err != nil { + return nil, err } } return rv, nil diff --git a/vendor/github.com/blevesearch/bleve/search/collector/list.go b/vendor/github.com/blevesearch/bleve/search/collector/list.go index 51b47b12..ec2f69cb 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/list.go +++ b/vendor/github.com/blevesearch/bleve/search/collector/list.go @@ -34,7 +34,16 @@ func newStoreList(cap int, compare collectorCompare) *collectStoreList { return rv } -func (c *collectStoreList) Add(doc *search.DocumentMatch) { +func (c *collectStoreList) AddNotExceedingSize(doc *search.DocumentMatch, + size int) *search.DocumentMatch { + c.add(doc) + if c.len() > size { + return c.removeLast() + } + return nil +} + +func (c *collectStoreList) add(doc *search.DocumentMatch) { for e := c.results.Front(); e != nil; e = e.Next() { curr := e.Value.(*search.DocumentMatch) if c.compare(doc, curr) >= 0 { @@ -46,7 +55,7 @@ func (c *collectStoreList) Add(doc *search.DocumentMatch) { c.results.PushBack(doc) } -func (c *collectStoreList) RemoveLast() *search.DocumentMatch { +func (c *collectStoreList) removeLast() *search.DocumentMatch { return c.results.Remove(c.results.Front()).(*search.DocumentMatch) } @@ -73,6 +82,6 @@ func (c *collectStoreList) Final(skip int, fixup collectorFixup) (search.Documen return search.DocumentMatchCollection{}, nil } -func (c *collectStoreList) Len() int { +func (c *collectStoreList) len() int { return c.results.Len() } diff --git a/vendor/github.com/blevesearch/bleve/search/collector/slice.go b/vendor/github.com/blevesearch/bleve/search/collector/slice.go index b061643e..32cb8624 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/slice.go +++ b/vendor/github.com/blevesearch/bleve/search/collector/slice.go @@ -29,7 +29,16 @@ func newStoreSlice(cap int, compare collectorCompare) *collectStoreSlice { return rv } -func (c *collectStoreSlice) Add(doc *search.DocumentMatch) { +func (c *collectStoreSlice) AddNotExceedingSize(doc *search.DocumentMatch, + size int) *search.DocumentMatch { + c.add(doc) + if c.len() > size { + return c.removeLast() + } + return nil +} + +func (c *collectStoreSlice) add(doc *search.DocumentMatch) { // find where to insert, starting at end (lowest) i := len(c.slice) for ; i > 0; i-- { @@ -44,7 +53,7 @@ func (c *collectStoreSlice) Add(doc *search.DocumentMatch) { c.slice[i] = doc } -func (c *collectStoreSlice) RemoveLast() *search.DocumentMatch { +func (c *collectStoreSlice) removeLast() *search.DocumentMatch { var rv *search.DocumentMatch rv, c.slice = c.slice[len(c.slice)-1], c.slice[:len(c.slice)-1] return rv @@ -63,6 +72,6 @@ func (c *collectStoreSlice) Final(skip int, fixup collectorFixup) (search.Docume return search.DocumentMatchCollection{}, nil } -func (c *collectStoreSlice) Len() int { +func (c *collectStoreSlice) len() int { return len(c.slice) } diff --git a/vendor/github.com/blevesearch/bleve/search/collector/topn.go b/vendor/github.com/blevesearch/bleve/search/collector/topn.go index 946ca3ec..4b2682da 100644 --- a/vendor/github.com/blevesearch/bleve/search/collector/topn.go +++ b/vendor/github.com/blevesearch/bleve/search/collector/topn.go @@ -15,13 +15,31 @@ package collector import ( + "context" + "reflect" "time" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" - "golang.org/x/net/context" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeTopNCollector int + +func init() { + var coll TopNCollector + reflectStaticSizeTopNCollector = int(reflect.TypeOf(coll).Size()) +} + +type collectorStore interface { + // Add the document, and if the new store size exceeds the provided size + // the last element is removed and returned. If the size has not been + // exceeded, nil is returned. + AddNotExceedingSize(doc *search.DocumentMatch, size int) *search.DocumentMatch + + Final(skip int, fixup collectorFixup) (search.DocumentMatchCollection, error) +} + // PreAllocSizeSkipCap will cap preallocation to this amount when // size+skip exceeds this value var PreAllocSizeSkipCap = 1000 @@ -41,7 +59,7 @@ type TopNCollector struct { results search.DocumentMatchCollection facetsBuilder *search.FacetsBuilder - store *collectStoreHeap + store collectorStore needDocIds bool neededFields []string @@ -49,6 +67,8 @@ type TopNCollector struct { cachedDesc []bool lowestMatchOutsideResults *search.DocumentMatch + updateFieldVisitor index.DocumentFieldTermVisitor + dvReader index.DocValueReader } // CheckDoneEvery controls how frequently we check the context deadline @@ -68,9 +88,15 @@ func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector backingSize = PreAllocSizeSkipCap + 1 } - hc.store = newStoreHeap(backingSize, func(i, j *search.DocumentMatch) int { - return hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, i, j) - }) + if size+skip > 10 { + hc.store = newStoreHeap(backingSize, func(i, j *search.DocumentMatch) int { + return hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, i, j) + }) + } else { + hc.store = newStoreSlice(backingSize, func(i, j *search.DocumentMatch) int { + return hc.sort.Compare(hc.cachedScoring, hc.cachedDesc, i, j) + }) + } // these lookups traverse an interface, so do once up-front if sort.RequiresDocID() { @@ -83,6 +109,22 @@ func NewTopNCollector(size int, skip int, sort search.SortOrder) *TopNCollector return hc } +func (hc *TopNCollector) Size() int { + sizeInBytes := reflectStaticSizeTopNCollector + size.SizeOfPtr + + if hc.facetsBuilder != nil { + sizeInBytes += hc.facetsBuilder.Size() + } + + for _, entry := range hc.neededFields { + sizeInBytes += len(entry) + size.SizeOfString + } + + sizeInBytes += len(hc.cachedScoring) + len(hc.cachedDesc) + + return sizeInBytes +} + // Collect goes to the index to find the matching documents func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, reader index.IndexReader) error { startTime := time.Now() @@ -100,6 +142,18 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, DocumentMatchPool: search.NewDocumentMatchPool(backingSize+searcher.DocumentMatchPoolSize(), len(hc.sort)), } + hc.dvReader, err = reader.DocValueReader(hc.neededFields) + if err != nil { + return err + } + + hc.updateFieldVisitor = func(field string, term []byte) { + if hc.facetsBuilder != nil { + hc.facetsBuilder.UpdateVisitor(field, term) + } + hc.sort.UpdateVisitor(field, term) + } + select { case <-ctx.Done(): return ctx.Err() @@ -184,9 +238,8 @@ func (hc *TopNCollector) collectSingle(ctx *search.SearchContext, reader index.I } } - hc.store.Add(d) - if hc.store.Len() > hc.size+hc.skip { - removed := hc.store.RemoveLast() + removed := hc.store.AddNotExceedingSize(d, hc.size+hc.skip) + if removed != nil { if hc.lowestMatchOutsideResults == nil { hc.lowestMatchOutsideResults = removed } else { @@ -209,13 +262,7 @@ func (hc *TopNCollector) visitFieldTerms(reader index.IndexReader, d *search.Doc hc.facetsBuilder.StartDoc() } - err := reader.DocumentVisitFieldTerms(d.IndexInternalID, hc.neededFields, func(field string, term []byte) { - if hc.facetsBuilder != nil { - hc.facetsBuilder.UpdateVisitor(field, term) - } - hc.sort.UpdateVisitor(field, term) - }) - + err := hc.dvReader.VisitDocValues(d.IndexInternalID, hc.updateFieldVisitor) if hc.facetsBuilder != nil { hc.facetsBuilder.EndDoc() } @@ -243,6 +290,7 @@ func (hc *TopNCollector) finalizeResults(r index.IndexReader) error { return err } } + doc.Complete(nil) return nil }) @@ -274,5 +322,5 @@ func (hc *TopNCollector) FacetResults() search.FacetResults { if hc.facetsBuilder != nil { return hc.facetsBuilder.Results() } - return search.FacetResults{} + return nil } diff --git a/vendor/github.com/blevesearch/bleve/search/explanation.go b/vendor/github.com/blevesearch/bleve/search/explanation.go index 766367d7..3b81737b 100644 --- a/vendor/github.com/blevesearch/bleve/search/explanation.go +++ b/vendor/github.com/blevesearch/bleve/search/explanation.go @@ -17,8 +17,18 @@ package search import ( "encoding/json" "fmt" + "reflect" + + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeExplanation int + +func init() { + var e Explanation + reflectStaticSizeExplanation = int(reflect.TypeOf(e).Size()) +} + type Explanation struct { Value float64 `json:"value"` Message string `json:"message"` @@ -32,3 +42,14 @@ func (expl *Explanation) String() string { } return string(js) } + +func (expl *Explanation) Size() int { + sizeInBytes := reflectStaticSizeExplanation + size.SizeOfPtr + + len(expl.Message) + + for _, entry := range expl.Children { + sizeInBytes += entry.Size() + } + + return sizeInBytes +} diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go index 8657a553..c45442e4 100644 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go +++ b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_datetime.go @@ -15,13 +15,25 @@ package facet import ( + "reflect" "sort" "time" "github.com/blevesearch/bleve/numeric" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeDateTimeFacetBuilder int +var reflectStaticSizedateTimeRange int + +func init() { + var dtfb DateTimeFacetBuilder + reflectStaticSizeDateTimeFacetBuilder = int(reflect.TypeOf(dtfb).Size()) + var dtr dateTimeRange + reflectStaticSizedateTimeRange = int(reflect.TypeOf(dtr).Size()) +} + type dateTimeRange struct { start time.Time end time.Time @@ -46,6 +58,23 @@ func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder { } } +func (fb *DateTimeFacetBuilder) Size() int { + sizeInBytes := reflectStaticSizeDateTimeFacetBuilder + size.SizeOfPtr + + len(fb.field) + + for k, _ := range fb.termsCount { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfInt + } + + for k, _ := range fb.ranges { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfPtr + reflectStaticSizedateTimeRange + } + + return sizeInBytes +} + func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) { r := dateTimeRange{ start: start, diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go index 2ab5f278..c1692b54 100644 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go +++ b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_numeric.go @@ -15,12 +15,24 @@ package facet import ( + "reflect" "sort" "github.com/blevesearch/bleve/numeric" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeNumericFacetBuilder int +var reflectStaticSizenumericRange int + +func init() { + var nfb NumericFacetBuilder + reflectStaticSizeNumericFacetBuilder = int(reflect.TypeOf(nfb).Size()) + var nr numericRange + reflectStaticSizenumericRange = int(reflect.TypeOf(nr).Size()) +} + type numericRange struct { min *float64 max *float64 @@ -45,6 +57,23 @@ func NewNumericFacetBuilder(field string, size int) *NumericFacetBuilder { } } +func (fb *NumericFacetBuilder) Size() int { + sizeInBytes := reflectStaticSizeNumericFacetBuilder + size.SizeOfPtr + + len(fb.field) + + for k, _ := range fb.termsCount { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfInt + } + + for k, _ := range fb.ranges { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfPtr + reflectStaticSizenumericRange + } + + return sizeInBytes +} + func (fb *NumericFacetBuilder) AddRange(name string, min, max *float64) { r := numericRange{ min: min, diff --git a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go index a41e475a..5b5901e0 100644 --- a/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go +++ b/vendor/github.com/blevesearch/bleve/search/facet/facet_builder_terms.go @@ -15,11 +15,20 @@ package facet import ( + "reflect" "sort" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeTermsFacetBuilder int + +func init() { + var tfb TermsFacetBuilder + reflectStaticSizeTermsFacetBuilder = int(reflect.TypeOf(tfb).Size()) +} + type TermsFacetBuilder struct { size int field string @@ -37,6 +46,18 @@ func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder { } } +func (fb *TermsFacetBuilder) Size() int { + sizeInBytes := reflectStaticSizeTermsFacetBuilder + size.SizeOfPtr + + len(fb.field) + + for k, _ := range fb.termsCount { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfInt + } + + return sizeInBytes +} + func (fb *TermsFacetBuilder) Field() string { return fb.field } diff --git a/vendor/github.com/blevesearch/bleve/search/facets_builder.go b/vendor/github.com/blevesearch/bleve/search/facets_builder.go index 05e27041..c5d41e2d 100644 --- a/vendor/github.com/blevesearch/bleve/search/facets_builder.go +++ b/vendor/github.com/blevesearch/bleve/search/facets_builder.go @@ -15,11 +15,32 @@ package search import ( + "reflect" "sort" "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeFacetsBuilder int +var reflectStaticSizeFacetResult int +var reflectStaticSizeTermFacet int +var reflectStaticSizeNumericRangeFacet int +var reflectStaticSizeDateRangeFacet int + +func init() { + var fb FacetsBuilder + reflectStaticSizeFacetsBuilder = int(reflect.TypeOf(fb).Size()) + var fr FacetResult + reflectStaticSizeFacetResult = int(reflect.TypeOf(fr).Size()) + var tf TermFacet + reflectStaticSizeTermFacet = int(reflect.TypeOf(tf).Size()) + var nrf NumericRangeFacet + reflectStaticSizeNumericRangeFacet = int(reflect.TypeOf(nrf).Size()) + var drf DateRangeFacet + reflectStaticSizeDateRangeFacet = int(reflect.TypeOf(drf).Size()) +} + type FacetBuilder interface { StartDoc() UpdateVisitor(field string, term []byte) @@ -27,6 +48,8 @@ type FacetBuilder interface { Result() *FacetResult Field() string + + Size() int } type FacetsBuilder struct { @@ -42,6 +65,21 @@ func NewFacetsBuilder(indexReader index.IndexReader) *FacetsBuilder { } } +func (fb *FacetsBuilder) Size() int { + sizeInBytes := reflectStaticSizeFacetsBuilder + size.SizeOfPtr + + for k, v := range fb.facets { + sizeInBytes += size.SizeOfString + len(k) + + v.Size() + } + + for _, entry := range fb.fields { + sizeInBytes += size.SizeOfString + len(entry) + } + + return sizeInBytes +} + func (fb *FacetsBuilder) Add(name string, facetBuilder FacetBuilder) { fb.facets[name] = facetBuilder fb.fields = append(fb.fields, facetBuilder.Field()) @@ -213,6 +251,14 @@ type FacetResult struct { DateRanges DateRangeFacets `json:"date_ranges,omitempty"` } +func (fr *FacetResult) Size() int { + return reflectStaticSizeFacetResult + size.SizeOfPtr + + len(fr.Field) + + len(fr.Terms)*(reflectStaticSizeTermFacet+size.SizeOfPtr) + + len(fr.NumericRanges)*(reflectStaticSizeNumericRangeFacet+size.SizeOfPtr) + + len(fr.DateRanges)*(reflectStaticSizeDateRangeFacet+size.SizeOfPtr) +} + func (fr *FacetResult) Merge(other *FacetResult) { fr.Total += other.Total fr.Missing += other.Missing diff --git a/vendor/github.com/blevesearch/bleve/search/levenshtein.go b/vendor/github.com/blevesearch/bleve/search/levenshtein.go index ec033143..687608d3 100644 --- a/vendor/github.com/blevesearch/bleve/search/levenshtein.go +++ b/vendor/github.com/blevesearch/bleve/search/levenshtein.go @@ -57,15 +57,24 @@ func LevenshteinDistance(a, b string) int { // in which case the first return val will be the max // and the second will be true, indicating max was exceeded func LevenshteinDistanceMax(a, b string, max int) (int, bool) { + v, wasMax, _ := LevenshteinDistanceMaxReuseSlice(a, b, max, nil) + return v, wasMax +} + +func LevenshteinDistanceMaxReuseSlice(a, b string, max int, d []int) (int, bool, []int) { la := len(a) lb := len(b) ld := int(math.Abs(float64(la - lb))) if ld > max { - return max, true + return max, true, d } - d := make([]int, la+1) + if cap(d) < la+1 { + d = make([]int, la+1) + } + d = d[:la+1] + var lastdiag, olddiag, temp int for i := 1; i <= la; i++ { @@ -98,8 +107,8 @@ func LevenshteinDistanceMax(a, b string, max int) (int, bool) { } // after each row if rowmin isn't less than max stop if rowmin > max { - return max, true + return max, true, d } } - return d[la], false + return d[la], false, d } diff --git a/vendor/github.com/blevesearch/bleve/search/pool.go b/vendor/github.com/blevesearch/bleve/search/pool.go index b9b52a61..ba8be8fc 100644 --- a/vendor/github.com/blevesearch/bleve/search/pool.go +++ b/vendor/github.com/blevesearch/bleve/search/pool.go @@ -14,6 +14,17 @@ package search +import ( + "reflect" +) + +var reflectStaticSizeDocumentMatchPool int + +func init() { + var dmp DocumentMatchPool + reflectStaticSizeDocumentMatchPool = int(reflect.TypeOf(dmp).Size()) +} + // DocumentMatchPoolTooSmall is a callback function that can be executed // when the DocumentMatchPool does not have sufficient capacity // By default we just perform just-in-time allocation, but you could log diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string.go b/vendor/github.com/blevesearch/bleve/search/query/query_string.go index 7d93d34b..ecafe6bc 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string.go +++ b/vendor/github.com/blevesearch/bleve/search/query/query_string.go @@ -43,6 +43,10 @@ func (q *QueryStringQuery) Boost() float64 { return q.BoostVal.Value() } +func (q *QueryStringQuery) Parse() (Query, error) { + return parseQuerySyntax(q.Query) +} + func (q *QueryStringQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { newQuery, err := parseQuerySyntax(q.Query) if err != nil { diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go index aad6f9c1..48cdf3ae 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go +++ b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_conjunction.go @@ -15,13 +15,27 @@ package scorer import ( + "reflect" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeConjunctionQueryScorer int + +func init() { + var cqs ConjunctionQueryScorer + reflectStaticSizeConjunctionQueryScorer = int(reflect.TypeOf(cqs).Size()) +} + type ConjunctionQueryScorer struct { options search.SearcherOptions } +func (s *ConjunctionQueryScorer) Size() int { + return reflectStaticSizeConjunctionQueryScorer + size.SizeOfPtr +} + func NewConjunctionQueryScorer(options search.SearcherOptions) *ConjunctionQueryScorer { return &ConjunctionQueryScorer{ options: options, @@ -35,15 +49,11 @@ func (s *ConjunctionQueryScorer) Score(ctx *search.SearchContext, constituents [ childrenExplanations = make([]*search.Explanation, len(constituents)) } - locations := []search.FieldTermLocationMap{} for i, docMatch := range constituents { sum += docMatch.Score if s.options.Explain { childrenExplanations[i] = docMatch.Expl } - if docMatch.Locations != nil { - locations = append(locations, docMatch.Locations) - } } newScore := sum var newExpl *search.Explanation @@ -55,11 +65,8 @@ func (s *ConjunctionQueryScorer) Score(ctx *search.SearchContext, constituents [ rv := constituents[0] rv.Score = newScore rv.Expl = newExpl - if len(locations) == 1 { - rv.Locations = locations[0] - } else if len(locations) > 1 { - rv.Locations = search.MergeLocations(locations) - } + rv.FieldTermLocations = search.MergeFieldTermLocations( + rv.FieldTermLocations, constituents[1:]) return rv } diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go index a65a826f..dc10fdaa 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go +++ b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_constant.go @@ -16,11 +16,20 @@ package scorer import ( "fmt" + "reflect" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeConstantScorer int + +func init() { + var cs ConstantScorer + reflectStaticSizeConstantScorer = int(reflect.TypeOf(cs).Size()) +} + type ConstantScorer struct { constant float64 boost float64 @@ -30,6 +39,16 @@ type ConstantScorer struct { queryWeightExplanation *search.Explanation } +func (s *ConstantScorer) Size() int { + sizeInBytes := reflectStaticSizeConstantScorer + size.SizeOfPtr + + if s.queryWeightExplanation != nil { + sizeInBytes += s.queryWeightExplanation.Size() + } + + return sizeInBytes +} + func NewConstantScorer(constant float64, boost float64, options search.SearcherOptions) *ConstantScorer { rv := ConstantScorer{ options: options, diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go index 184a15d2..7a955e16 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go +++ b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_disjunction.go @@ -16,14 +16,27 @@ package scorer import ( "fmt" + "reflect" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeDisjunctionQueryScorer int + +func init() { + var dqs DisjunctionQueryScorer + reflectStaticSizeDisjunctionQueryScorer = int(reflect.TypeOf(dqs).Size()) +} + type DisjunctionQueryScorer struct { options search.SearcherOptions } +func (s *DisjunctionQueryScorer) Size() int { + return reflectStaticSizeDisjunctionQueryScorer + size.SizeOfPtr +} + func NewDisjunctionQueryScorer(options search.SearcherOptions) *DisjunctionQueryScorer { return &DisjunctionQueryScorer{ options: options, @@ -37,15 +50,11 @@ func (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents [ childrenExplanations = make([]*search.Explanation, len(constituents)) } - var locations []search.FieldTermLocationMap for i, docMatch := range constituents { sum += docMatch.Score if s.options.Explain { childrenExplanations[i] = docMatch.Expl } - if docMatch.Locations != nil { - locations = append(locations, docMatch.Locations) - } } var rawExpl *search.Explanation @@ -67,11 +76,8 @@ func (s *DisjunctionQueryScorer) Score(ctx *search.SearchContext, constituents [ rv := constituents[0] rv.Score = newScore rv.Expl = newExpl - if len(locations) == 1 { - rv.Locations = locations[0] - } else if len(locations) > 1 { - rv.Locations = search.MergeLocations(locations) - } + rv.FieldTermLocations = search.MergeFieldTermLocations( + rv.FieldTermLocations, constituents[1:]) return rv } diff --git a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go index b5f46322..5544f2d0 100644 --- a/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go +++ b/vendor/github.com/blevesearch/bleve/search/scorer/scorer_term.go @@ -17,13 +17,22 @@ package scorer import ( "fmt" "math" + "reflect" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeTermQueryScorer int + +func init() { + var tqs TermQueryScorer + reflectStaticSizeTermQueryScorer = int(reflect.TypeOf(tqs).Size()) +} + type TermQueryScorer struct { - queryTerm []byte + queryTerm string queryField string queryBoost float64 docTerm uint64 @@ -36,9 +45,24 @@ type TermQueryScorer struct { queryWeightExplanation *search.Explanation } +func (s *TermQueryScorer) Size() int { + sizeInBytes := reflectStaticSizeTermQueryScorer + size.SizeOfPtr + + len(s.queryTerm) + len(s.queryField) + + if s.idfExplanation != nil { + sizeInBytes += s.idfExplanation.Size() + } + + if s.queryWeightExplanation != nil { + sizeInBytes += s.queryWeightExplanation.Size() + } + + return sizeInBytes +} + func NewTermQueryScorer(queryTerm []byte, queryField string, queryBoost float64, docTotal, docTerm uint64, options search.SearcherOptions) *TermQueryScorer { rv := TermQueryScorer{ - queryTerm: queryTerm, + queryTerm: string(queryTerm), queryField: queryField, queryBoost: queryBoost, docTerm: docTerm, @@ -82,7 +106,7 @@ func (s *TermQueryScorer) SetQueryNorm(qnorm float64) { } s.queryWeightExplanation = &search.Explanation{ Value: s.queryWeight, - Message: fmt.Sprintf("queryWeight(%s:%s^%f), product of:", s.queryField, string(s.queryTerm), s.queryBoost), + Message: fmt.Sprintf("queryWeight(%s:%s^%f), product of:", s.queryField, s.queryTerm, s.queryBoost), Children: childrenExplanations, } } @@ -104,7 +128,7 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term childrenExplanations := make([]*search.Explanation, 3) childrenExplanations[0] = &search.Explanation{ Value: tf, - Message: fmt.Sprintf("tf(termFreq(%s:%s)=%d", s.queryField, string(s.queryTerm), termMatch.Freq), + Message: fmt.Sprintf("tf(termFreq(%s:%s)=%d", s.queryField, s.queryTerm, termMatch.Freq), } childrenExplanations[1] = &search.Explanation{ Value: termMatch.Norm, @@ -113,7 +137,7 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term childrenExplanations[2] = s.idfExplanation scoreExplanation = &search.Explanation{ Value: score, - Message: fmt.Sprintf("fieldWeight(%s:%s in %s), product of:", s.queryField, string(s.queryTerm), termMatch.ID), + Message: fmt.Sprintf("fieldWeight(%s:%s in %s), product of:", s.queryField, s.queryTerm, termMatch.ID), Children: childrenExplanations, } } @@ -127,7 +151,7 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term childExplanations[1] = scoreExplanation scoreExplanation = &search.Explanation{ Value: score, - Message: fmt.Sprintf("weight(%s:%s^%f in %s), product of:", s.queryField, string(s.queryTerm), s.queryBoost, termMatch.ID), + Message: fmt.Sprintf("weight(%s:%s^%f in %s), product of:", s.queryField, s.queryTerm, s.queryBoost, termMatch.ID), Children: childExplanations, } } @@ -140,41 +164,31 @@ func (s *TermQueryScorer) Score(ctx *search.SearchContext, termMatch *index.Term rv.Expl = scoreExplanation } - if termMatch.Vectors != nil && len(termMatch.Vectors) > 0 { - locs := make([]search.Location, len(termMatch.Vectors)) - locsUsed := 0 - - totalPositions := 0 - for _, v := range termMatch.Vectors { - totalPositions += len(v.ArrayPositions) + if len(termMatch.Vectors) > 0 { + if cap(rv.FieldTermLocations) < len(termMatch.Vectors) { + rv.FieldTermLocations = make([]search.FieldTermLocation, 0, len(termMatch.Vectors)) } - positions := make(search.ArrayPositions, totalPositions) - positionsUsed := 0 - rv.Locations = make(search.FieldTermLocationMap) for _, v := range termMatch.Vectors { - tlm := rv.Locations[v.Field] - if tlm == nil { - tlm = make(search.TermLocationMap) - rv.Locations[v.Field] = tlm - } - - loc := &locs[locsUsed] - locsUsed++ - - loc.Pos = v.Pos - loc.Start = v.Start - loc.End = v.End - + var ap search.ArrayPositions if len(v.ArrayPositions) > 0 { - loc.ArrayPositions = positions[positionsUsed : positionsUsed+len(v.ArrayPositions)] - for i, ap := range v.ArrayPositions { - loc.ArrayPositions[i] = ap + n := len(rv.FieldTermLocations) + if n < cap(rv.FieldTermLocations) { // reuse ap slice if available + ap = rv.FieldTermLocations[:n+1][n].Location.ArrayPositions[:0] } - positionsUsed += len(v.ArrayPositions) + ap = append(ap, v.ArrayPositions...) } - - tlm[string(s.queryTerm)] = append(tlm[string(s.queryTerm)], loc) + rv.FieldTermLocations = + append(rv.FieldTermLocations, search.FieldTermLocation{ + Field: v.Field, + Term: s.queryTerm, + Location: search.Location{ + Pos: v.Pos, + Start: v.Start, + End: v.End, + ArrayPositions: ap, + }, + }) } } diff --git a/vendor/github.com/blevesearch/bleve/search/search.go b/vendor/github.com/blevesearch/bleve/search/search.go index a9bf9db9..440c0957 100644 --- a/vendor/github.com/blevesearch/bleve/search/search.go +++ b/vendor/github.com/blevesearch/bleve/search/search.go @@ -16,11 +16,26 @@ package search import ( "fmt" + "reflect" "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeDocumentMatch int +var reflectStaticSizeSearchContext int +var reflectStaticSizeLocation int + +func init() { + var dm DocumentMatch + reflectStaticSizeDocumentMatch = int(reflect.TypeOf(dm).Size()) + var sc SearchContext + reflectStaticSizeSearchContext = int(reflect.TypeOf(sc).Size()) + var l Location + reflectStaticSizeLocation = int(reflect.TypeOf(l).Size()) +} + type ArrayPositions []uint64 func (ap ArrayPositions) Equals(other ArrayPositions) bool { @@ -36,12 +51,22 @@ func (ap ArrayPositions) Equals(other ArrayPositions) bool { } type Location struct { - Pos uint64 `json:"pos"` - Start uint64 `json:"start"` - End uint64 `json:"end"` + // Pos is the position of the term within the field, starting at 1 + Pos uint64 `json:"pos"` + + // Start and End are the byte offsets of the term in the field + Start uint64 `json:"start"` + End uint64 `json:"end"` + + // ArrayPositions contains the positions of the term within any elements. ArrayPositions ArrayPositions `json:"array_positions"` } +func (l *Location) Size() int { + return reflectStaticSizeLocation + size.SizeOfPtr + + len(l.ArrayPositions)*size.SizeOfUint64 +} + type Locations []*Location type TermLocationMap map[string]Locations @@ -52,6 +77,12 @@ func (t TermLocationMap) AddLocation(term string, location *Location) { type FieldTermLocationMap map[string]TermLocationMap +type FieldTermLocation struct { + Field string + Term string + Location Location +} + type FieldFragmentMap map[string][]string type DocumentMatch struct { @@ -74,6 +105,12 @@ type DocumentMatch struct { // used to maintain natural index order HitNumber uint64 `json:"-"` + + // used to temporarily hold field term location information during + // search processing in an efficient, recycle-friendly manner, to + // be later incorporated into the Locations map when search + // results are completed + FieldTermLocations []FieldTermLocation `json:"-"` } func (dm *DocumentMatch) AddFieldValue(name string, value interface{}) { @@ -103,15 +140,120 @@ func (dm *DocumentMatch) Reset() *DocumentMatch { indexInternalID := dm.IndexInternalID // remember the []interface{} used for sort sort := dm.Sort + // remember the FieldTermLocations backing array + ftls := dm.FieldTermLocations + for i := range ftls { // recycle the ArrayPositions of each location + ftls[i].Location.ArrayPositions = ftls[i].Location.ArrayPositions[:0] + } // idiom to copy over from empty DocumentMatch (0 allocations) *dm = DocumentMatch{} // reuse the []byte already allocated (and reset len to 0) dm.IndexInternalID = indexInternalID[:0] // reuse the []interface{} already allocated (and reset len to 0) dm.Sort = sort[:0] + // reuse the FieldTermLocations already allocated (and reset len to 0) + dm.FieldTermLocations = ftls[:0] return dm } +func (dm *DocumentMatch) Size() int { + sizeInBytes := reflectStaticSizeDocumentMatch + size.SizeOfPtr + + len(dm.Index) + + len(dm.ID) + + len(dm.IndexInternalID) + + if dm.Expl != nil { + sizeInBytes += dm.Expl.Size() + } + + for k, v := range dm.Locations { + sizeInBytes += size.SizeOfString + len(k) + for k1, v1 := range v { + sizeInBytes += size.SizeOfString + len(k1) + + size.SizeOfSlice + for _, entry := range v1 { + sizeInBytes += entry.Size() + } + } + } + + for k, v := range dm.Fragments { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfSlice + + for _, entry := range v { + sizeInBytes += size.SizeOfString + len(entry) + } + } + + for _, entry := range dm.Sort { + sizeInBytes += size.SizeOfString + len(entry) + } + + for k, _ := range dm.Fields { + sizeInBytes += size.SizeOfString + len(k) + + size.SizeOfPtr + } + + if dm.Document != nil { + sizeInBytes += dm.Document.Size() + } + + return sizeInBytes +} + +// Complete performs final preparation & transformation of the +// DocumentMatch at the end of search processing, also allowing the +// caller to provide an optional preallocated locations slice +func (dm *DocumentMatch) Complete(prealloc []Location) []Location { + // transform the FieldTermLocations slice into the Locations map + nlocs := len(dm.FieldTermLocations) + if nlocs > 0 { + if cap(prealloc) < nlocs { + prealloc = make([]Location, nlocs) + } + prealloc = prealloc[:nlocs] + + var lastField string + var tlm TermLocationMap + + for i, ftl := range dm.FieldTermLocations { + if lastField != ftl.Field { + lastField = ftl.Field + + if dm.Locations == nil { + dm.Locations = make(FieldTermLocationMap) + } + + tlm = dm.Locations[ftl.Field] + if tlm == nil { + tlm = make(TermLocationMap) + dm.Locations[ftl.Field] = tlm + } + } + + loc := &prealloc[i] + *loc = ftl.Location + + if len(loc.ArrayPositions) > 0 { // copy + loc.ArrayPositions = append(ArrayPositions(nil), loc.ArrayPositions...) + } + + tlm[ftl.Term] = append(tlm[ftl.Term], loc) + + dm.FieldTermLocations[i] = FieldTermLocation{ // recycle + Location: Location{ + ArrayPositions: ftl.Location.ArrayPositions[:0], + }, + } + } + } + + dm.FieldTermLocations = dm.FieldTermLocations[:0] // recycle + + return prealloc +} + func (dm *DocumentMatch) String() string { return fmt.Sprintf("[%s-%f]", string(dm.IndexInternalID), dm.Score) } @@ -130,6 +272,7 @@ type Searcher interface { SetQueryNorm(float64) Count() uint64 Min() int + Size() int DocumentMatchPoolSize() int } @@ -143,3 +286,18 @@ type SearcherOptions struct { type SearchContext struct { DocumentMatchPool *DocumentMatchPool } + +func (sc *SearchContext) Size() int { + sizeInBytes := reflectStaticSizeSearchContext + size.SizeOfPtr + + reflectStaticSizeDocumentMatchPool + size.SizeOfPtr + + if sc.DocumentMatchPool != nil { + for _, entry := range sc.DocumentMatchPool.avail { + if entry != nil { + sizeInBytes += entry.Size() + } + } + } + + return sizeInBytes +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go index a905c29e..f7ee2cd8 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_boolean.go @@ -16,12 +16,21 @@ package searcher import ( "math" + "reflect" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search/scorer" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeBooleanSearcher int + +func init() { + var bs BooleanSearcher + reflectStaticSizeBooleanSearcher = int(reflect.TypeOf(bs).Size()) +} + type BooleanSearcher struct { indexReader index.IndexReader mustSearcher search.Searcher @@ -52,6 +61,32 @@ func NewBooleanSearcher(indexReader index.IndexReader, mustSearcher search.Searc return &rv, nil } +func (s *BooleanSearcher) Size() int { + sizeInBytes := reflectStaticSizeBooleanSearcher + size.SizeOfPtr + + if s.mustSearcher != nil { + sizeInBytes += s.mustSearcher.Size() + } + + if s.shouldSearcher != nil { + sizeInBytes += s.shouldSearcher.Size() + } + + if s.mustNotSearcher != nil { + sizeInBytes += s.mustNotSearcher.Size() + } + + sizeInBytes += s.scorer.Size() + + for _, entry := range s.matches { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + return sizeInBytes +} + func (s *BooleanSearcher) computeQueryNorm() { // first calculate sum of squared weights sumOfSquaredWeights := 0.0 diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go index 9ab0e7fa..a4805267 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_conjunction.go @@ -16,13 +16,22 @@ package searcher import ( "math" + "reflect" "sort" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search/scorer" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeConjunctionSearcher int + +func init() { + var cs ConjunctionSearcher + reflectStaticSizeConjunctionSearcher = int(reflect.TypeOf(cs).Size()) +} + type ConjunctionSearcher struct { indexReader index.IndexReader searchers OrderedSearcherList @@ -51,31 +60,72 @@ func NewConjunctionSearcher(indexReader index.IndexReader, qsearchers []search.S scorer: scorer.NewConjunctionQueryScorer(options), } rv.computeQueryNorm() + + // attempt push-down conjunction optimization when there's >1 searchers + if len(searchers) > 1 { + var octx index.OptimizableContext + + for _, searcher := range searchers { + o, ok := searcher.(index.Optimizable) + if ok { + var err error + octx, err = o.Optimize("conjunction", octx) + if err != nil { + return nil, err + } + } + } + + if octx != nil { + err := octx.Finish() + if err != nil { + return nil, err + } + } + } + return &rv, nil } +func (s *ConjunctionSearcher) Size() int { + sizeInBytes := reflectStaticSizeConjunctionSearcher + size.SizeOfPtr + + s.scorer.Size() + + for _, entry := range s.searchers { + sizeInBytes += entry.Size() + } + + for _, entry := range s.currs { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + return sizeInBytes +} + func (s *ConjunctionSearcher) computeQueryNorm() { // first calculate sum of squared weights sumOfSquaredWeights := 0.0 - for _, termSearcher := range s.searchers { - sumOfSquaredWeights += termSearcher.Weight() + for _, searcher := range s.searchers { + sumOfSquaredWeights += searcher.Weight() } // now compute query norm from this s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) // finally tell all the downstream searchers the norm - for _, termSearcher := range s.searchers { - termSearcher.SetQueryNorm(s.queryNorm) + for _, searcher := range s.searchers { + searcher.SetQueryNorm(s.queryNorm) } } func (s *ConjunctionSearcher) initSearchers(ctx *search.SearchContext) error { var err error // get all searchers pointing at their first match - for i, termSearcher := range s.searchers { + for i, searcher := range s.searchers { if s.currs[i] != nil { ctx.DocumentMatchPool.Put(s.currs[i]) } - s.currs[i], err = termSearcher.Next(ctx) + s.currs[i], err = searcher.Next(ctx) if err != nil { return err } @@ -160,11 +210,11 @@ OUTER: // we know all the searchers are pointing at the same thing // so they all need to be bumped - for i, termSearcher := range s.searchers { + for i, searcher := range s.searchers { if s.currs[i] != rv { ctx.DocumentMatchPool.Put(s.currs[i]) } - s.currs[i], err = termSearcher.Next(ctx) + s.currs[i], err = searcher.Next(ctx) if err != nil { return nil, err } @@ -184,6 +234,9 @@ func (s *ConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexI } } for i := range s.searchers { + if s.currs[i] != nil && s.currs[i].IndexInternalID.Compare(ID) >= 0 { + continue + } err := s.advanceChild(ctx, i, ID) if err != nil { return nil, err diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go index 96bd5447..bbf7b4bb 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014 Couchbase, Inc. +// Copyright (c) 2018 Couchbase, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,12 +16,9 @@ package searcher import ( "fmt" - "math" - "sort" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" - "github.com/blevesearch/bleve/search/scorer" ) // DisjunctionMaxClauseCount is a compile time setting that applications can @@ -29,17 +26,26 @@ import ( // error instead of exeucting searches when the size exceeds this value. var DisjunctionMaxClauseCount = 0 -type DisjunctionSearcher struct { - indexReader index.IndexReader - searchers OrderedSearcherList - numSearchers int - queryNorm float64 - currs []*search.DocumentMatch - scorer *scorer.DisjunctionQueryScorer - min int - matching []*search.DocumentMatch - matchingIdxs []int - initialized bool +// DisjunctionHeapTakeover is a compile time setting that applications can +// adjust to control when the DisjunctionSearcher will switch from a simple +// slice implementation to a heap implementation. +var DisjunctionHeapTakeover = 10 + +func NewDisjunctionSearcher(indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions) ( + search.Searcher, error) { + return newDisjunctionSearcher(indexReader, qsearchers, min, options, true) +} + +func newDisjunctionSearcher(indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions, + limit bool) (search.Searcher, error) { + if len(qsearchers) > DisjunctionHeapTakeover { + return newDisjunctionHeapSearcher(indexReader, qsearchers, min, options, + limit) + } + return newDisjunctionSliceSearcher(indexReader, qsearchers, min, options, + limit) } func tooManyClauses(count int) bool { @@ -53,219 +59,3 @@ func tooManyClausesErr() error { return fmt.Errorf("TooManyClauses[maxClauseCount is set to %d]", DisjunctionMaxClauseCount) } - -func NewDisjunctionSearcher(indexReader index.IndexReader, - qsearchers []search.Searcher, min float64, options search.SearcherOptions) ( - *DisjunctionSearcher, error) { - return newDisjunctionSearcher(indexReader, qsearchers, min, options, - true) -} - -func newDisjunctionSearcher(indexReader index.IndexReader, - qsearchers []search.Searcher, min float64, options search.SearcherOptions, - limit bool) ( - *DisjunctionSearcher, error) { - if limit && tooManyClauses(len(qsearchers)) { - return nil, tooManyClausesErr() - } - // build the downstream searchers - searchers := make(OrderedSearcherList, len(qsearchers)) - for i, searcher := range qsearchers { - searchers[i] = searcher - } - // sort the searchers - sort.Sort(sort.Reverse(searchers)) - // build our searcher - rv := DisjunctionSearcher{ - indexReader: indexReader, - searchers: searchers, - numSearchers: len(searchers), - currs: make([]*search.DocumentMatch, len(searchers)), - scorer: scorer.NewDisjunctionQueryScorer(options), - min: int(min), - matching: make([]*search.DocumentMatch, len(searchers)), - matchingIdxs: make([]int, len(searchers)), - } - rv.computeQueryNorm() - return &rv, nil -} - -func (s *DisjunctionSearcher) computeQueryNorm() { - // first calculate sum of squared weights - sumOfSquaredWeights := 0.0 - for _, termSearcher := range s.searchers { - sumOfSquaredWeights += termSearcher.Weight() - } - // now compute query norm from this - s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) - // finally tell all the downstream searchers the norm - for _, termSearcher := range s.searchers { - termSearcher.SetQueryNorm(s.queryNorm) - } -} - -func (s *DisjunctionSearcher) initSearchers(ctx *search.SearchContext) error { - var err error - // get all searchers pointing at their first match - for i, termSearcher := range s.searchers { - if s.currs[i] != nil { - ctx.DocumentMatchPool.Put(s.currs[i]) - } - s.currs[i], err = termSearcher.Next(ctx) - if err != nil { - return err - } - } - - err = s.updateMatches() - if err != nil { - return err - } - - s.initialized = true - return nil -} - -func (s *DisjunctionSearcher) updateMatches() error { - matching := s.matching[:0] - matchingIdxs := s.matchingIdxs[:0] - - for i := 0; i < len(s.currs); i++ { - curr := s.currs[i] - if curr == nil { - continue - } - - if len(matching) > 0 { - cmp := curr.IndexInternalID.Compare(matching[0].IndexInternalID) - if cmp > 0 { - continue - } - - if cmp < 0 { - matching = matching[:0] - matchingIdxs = matchingIdxs[:0] - } - } - - matching = append(matching, curr) - matchingIdxs = append(matchingIdxs, i) - } - - s.matching = matching - s.matchingIdxs = matchingIdxs - - return nil -} - -func (s *DisjunctionSearcher) Weight() float64 { - var rv float64 - for _, searcher := range s.searchers { - rv += searcher.Weight() - } - return rv -} - -func (s *DisjunctionSearcher) SetQueryNorm(qnorm float64) { - for _, searcher := range s.searchers { - searcher.SetQueryNorm(qnorm) - } -} - -func (s *DisjunctionSearcher) Next(ctx *search.SearchContext) ( - *search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers(ctx) - if err != nil { - return nil, err - } - } - var err error - var rv *search.DocumentMatch - - found := false - for !found && len(s.matching) > 0 { - if len(s.matching) >= s.min { - found = true - // score this match - rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) - } - - // invoke next on all the matching searchers - for _, i := range s.matchingIdxs { - searcher := s.searchers[i] - if s.currs[i] != rv { - ctx.DocumentMatchPool.Put(s.currs[i]) - } - s.currs[i], err = searcher.Next(ctx) - if err != nil { - return nil, err - } - } - - err = s.updateMatches() - if err != nil { - return nil, err - } - } - return rv, nil -} - -func (s *DisjunctionSearcher) Advance(ctx *search.SearchContext, - ID index.IndexInternalID) (*search.DocumentMatch, error) { - if !s.initialized { - err := s.initSearchers(ctx) - if err != nil { - return nil, err - } - } - // get all searchers pointing at their first match - var err error - for i, termSearcher := range s.searchers { - if s.currs[i] != nil { - ctx.DocumentMatchPool.Put(s.currs[i]) - } - s.currs[i], err = termSearcher.Advance(ctx, ID) - if err != nil { - return nil, err - } - } - - err = s.updateMatches() - if err != nil { - return nil, err - } - - return s.Next(ctx) -} - -func (s *DisjunctionSearcher) Count() uint64 { - // for now return a worst case - var sum uint64 - for _, searcher := range s.searchers { - sum += searcher.Count() - } - return sum -} - -func (s *DisjunctionSearcher) Close() (rv error) { - for _, searcher := range s.searchers { - err := searcher.Close() - if err != nil && rv == nil { - rv = err - } - } - return rv -} - -func (s *DisjunctionSearcher) Min() int { - return s.min -} - -func (s *DisjunctionSearcher) DocumentMatchPoolSize() int { - rv := len(s.currs) - for _, s := range s.searchers { - rv += s.DocumentMatchPoolSize() - } - return rv -} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go new file mode 100644 index 00000000..ffa373d2 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_heap.go @@ -0,0 +1,343 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "bytes" + "container/heap" + "math" + "reflect" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/scorer" + "github.com/blevesearch/bleve/size" +) + +var reflectStaticSizeDisjunctionHeapSearcher int +var reflectStaticSizeSearcherCurr int + +func init() { + var dhs DisjunctionHeapSearcher + reflectStaticSizeDisjunctionHeapSearcher = int(reflect.TypeOf(dhs).Size()) + + var sc SearcherCurr + reflectStaticSizeSearcherCurr = int(reflect.TypeOf(sc).Size()) +} + +type SearcherCurr struct { + searcher search.Searcher + curr *search.DocumentMatch +} + +type DisjunctionHeapSearcher struct { + indexReader index.IndexReader + + numSearchers int + scorer *scorer.DisjunctionQueryScorer + min int + queryNorm float64 + initialized bool + searchers []search.Searcher + heap []*SearcherCurr + + matching []*search.DocumentMatch + matchingCurrs []*SearcherCurr +} + +func newDisjunctionHeapSearcher(indexReader index.IndexReader, + searchers []search.Searcher, min float64, options search.SearcherOptions, + limit bool) ( + *DisjunctionHeapSearcher, error) { + if limit && tooManyClauses(len(searchers)) { + return nil, tooManyClausesErr() + } + + // build our searcher + rv := DisjunctionHeapSearcher{ + indexReader: indexReader, + searchers: searchers, + numSearchers: len(searchers), + scorer: scorer.NewDisjunctionQueryScorer(options), + min: int(min), + matching: make([]*search.DocumentMatch, len(searchers)), + matchingCurrs: make([]*SearcherCurr, len(searchers)), + heap: make([]*SearcherCurr, 0, len(searchers)), + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *DisjunctionHeapSearcher) Size() int { + sizeInBytes := reflectStaticSizeDisjunctionHeapSearcher + size.SizeOfPtr + + s.scorer.Size() + + for _, entry := range s.searchers { + sizeInBytes += entry.Size() + } + + for _, entry := range s.matching { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + // for matchingCurrs and heap, just use static size * len + // since searchers and document matches already counted above + sizeInBytes += len(s.matchingCurrs) * reflectStaticSizeSearcherCurr + sizeInBytes += len(s.heap) * reflectStaticSizeSearcherCurr + + return sizeInBytes +} + +func (s *DisjunctionHeapSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + for _, searcher := range s.searchers { + sumOfSquaredWeights += searcher.Weight() + } + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + for _, searcher := range s.searchers { + searcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *DisjunctionHeapSearcher) initSearchers(ctx *search.SearchContext) error { + // alloc a single block of SearcherCurrs + block := make([]SearcherCurr, len(s.searchers)) + + // get all searchers pointing at their first match + for i, searcher := range s.searchers { + curr, err := searcher.Next(ctx) + if err != nil { + return err + } + if curr != nil { + block[i].searcher = searcher + block[i].curr = curr + heap.Push(s, &block[i]) + } + } + + err := s.updateMatches() + if err != nil { + return err + } + s.initialized = true + return nil +} + +func (s *DisjunctionHeapSearcher) updateMatches() error { + matching := s.matching[:0] + matchingCurrs := s.matchingCurrs[:0] + + if len(s.heap) > 0 { + + // top of the heap is our next hit + next := heap.Pop(s).(*SearcherCurr) + matching = append(matching, next.curr) + matchingCurrs = append(matchingCurrs, next) + + // now as long as top of heap matches, keep popping + for len(s.heap) > 0 && bytes.Compare(next.curr.IndexInternalID, s.heap[0].curr.IndexInternalID) == 0 { + next = heap.Pop(s).(*SearcherCurr) + matching = append(matching, next.curr) + matchingCurrs = append(matchingCurrs, next) + } + } + + s.matching = matching + s.matchingCurrs = matchingCurrs + + return nil +} + +func (s *DisjunctionHeapSearcher) Weight() float64 { + var rv float64 + for _, searcher := range s.searchers { + rv += searcher.Weight() + } + return rv +} + +func (s *DisjunctionHeapSearcher) SetQueryNorm(qnorm float64) { + for _, searcher := range s.searchers { + searcher.SetQueryNorm(qnorm) + } +} + +func (s *DisjunctionHeapSearcher) Next(ctx *search.SearchContext) ( + *search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + var rv *search.DocumentMatch + found := false + for !found && len(s.matching) > 0 { + if len(s.matching) >= s.min { + found = true + // score this match + rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + } + + // invoke next on all the matching searchers + for _, matchingCurr := range s.matchingCurrs { + if matchingCurr.curr != rv { + ctx.DocumentMatchPool.Put(matchingCurr.curr) + } + curr, err := matchingCurr.searcher.Next(ctx) + if err != nil { + return nil, err + } + if curr != nil { + matchingCurr.curr = curr + heap.Push(s, matchingCurr) + } + } + + err := s.updateMatches() + if err != nil { + return nil, err + } + } + + return rv, nil +} + +func (s *DisjunctionHeapSearcher) Advance(ctx *search.SearchContext, + ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + + // if there is anything in matching, toss it back onto the heap + for _, matchingCurr := range s.matchingCurrs { + heap.Push(s, matchingCurr) + } + s.matching = s.matching[:0] + s.matchingCurrs = s.matchingCurrs[:0] + + // find all searchers that actually need to be advanced + // advance them, using s.matchingCurrs as temp storage + for len(s.heap) > 0 && bytes.Compare(s.heap[0].curr.IndexInternalID, ID) < 0 { + searcherCurr := heap.Pop(s).(*SearcherCurr) + ctx.DocumentMatchPool.Put(searcherCurr.curr) + curr, err := searcherCurr.searcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + if curr != nil { + searcherCurr.curr = curr + s.matchingCurrs = append(s.matchingCurrs, searcherCurr) + } + } + // now all of the searchers that we advanced have to be pushed back + for _, matchingCurr := range s.matchingCurrs { + heap.Push(s, matchingCurr) + } + // reset our temp space + s.matchingCurrs = s.matchingCurrs[:0] + + err := s.updateMatches() + if err != nil { + return nil, err + } + + return s.Next(ctx) +} + +func (s *DisjunctionHeapSearcher) Count() uint64 { + // for now return a worst case + var sum uint64 + for _, searcher := range s.searchers { + sum += searcher.Count() + } + return sum +} + +func (s *DisjunctionHeapSearcher) Close() (rv error) { + for _, searcher := range s.searchers { + err := searcher.Close() + if err != nil && rv == nil { + rv = err + } + } + return rv +} + +func (s *DisjunctionHeapSearcher) Min() int { + return s.min +} + +func (s *DisjunctionHeapSearcher) DocumentMatchPoolSize() int { + rv := len(s.searchers) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} + +// a disjunction searcher implements the index.Optimizable interface +// but only activates on an edge case where the disjunction is a +// wrapper around a single Optimizable child searcher +func (s *DisjunctionHeapSearcher) Optimize(kind string, octx index.OptimizableContext) ( + index.OptimizableContext, error) { + if len(s.searchers) == 1 { + o, ok := s.searchers[0].(index.Optimizable) + if ok { + return o.Optimize(kind, octx) + } + } + + return octx, nil +} + +// heap impl + +func (s *DisjunctionHeapSearcher) Len() int { return len(s.heap) } + +func (s *DisjunctionHeapSearcher) Less(i, j int) bool { + if s.heap[i].curr == nil { + return true + } else if s.heap[j].curr == nil { + return false + } + return bytes.Compare(s.heap[i].curr.IndexInternalID, s.heap[j].curr.IndexInternalID) < 0 +} + +func (s *DisjunctionHeapSearcher) Swap(i, j int) { + s.heap[i], s.heap[j] = s.heap[j], s.heap[i] +} + +func (s *DisjunctionHeapSearcher) Push(x interface{}) { + s.heap = append(s.heap, x.(*SearcherCurr)) +} + +func (s *DisjunctionHeapSearcher) Pop() interface{} { + old := s.heap + n := len(old) + x := old[n-1] + s.heap = old[0 : n-1] + return x +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go new file mode 100644 index 00000000..e3efdf2a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_disjunction_slice.go @@ -0,0 +1,298 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package searcher + +import ( + "math" + "reflect" + "sort" + + "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/search/scorer" + "github.com/blevesearch/bleve/size" +) + +var reflectStaticSizeDisjunctionSliceSearcher int + +func init() { + var ds DisjunctionSliceSearcher + reflectStaticSizeDisjunctionSliceSearcher = int(reflect.TypeOf(ds).Size()) +} + +type DisjunctionSliceSearcher struct { + indexReader index.IndexReader + searchers OrderedSearcherList + numSearchers int + queryNorm float64 + currs []*search.DocumentMatch + scorer *scorer.DisjunctionQueryScorer + min int + matching []*search.DocumentMatch + matchingIdxs []int + initialized bool +} + +func newDisjunctionSliceSearcher(indexReader index.IndexReader, + qsearchers []search.Searcher, min float64, options search.SearcherOptions, + limit bool) ( + *DisjunctionSliceSearcher, error) { + if limit && tooManyClauses(len(qsearchers)) { + return nil, tooManyClausesErr() + } + // build the downstream searchers + searchers := make(OrderedSearcherList, len(qsearchers)) + for i, searcher := range qsearchers { + searchers[i] = searcher + } + // sort the searchers + sort.Sort(sort.Reverse(searchers)) + // build our searcher + rv := DisjunctionSliceSearcher{ + indexReader: indexReader, + searchers: searchers, + numSearchers: len(searchers), + currs: make([]*search.DocumentMatch, len(searchers)), + scorer: scorer.NewDisjunctionQueryScorer(options), + min: int(min), + matching: make([]*search.DocumentMatch, len(searchers)), + matchingIdxs: make([]int, len(searchers)), + } + rv.computeQueryNorm() + return &rv, nil +} + +func (s *DisjunctionSliceSearcher) Size() int { + sizeInBytes := reflectStaticSizeDisjunctionSliceSearcher + size.SizeOfPtr + + s.scorer.Size() + + for _, entry := range s.searchers { + sizeInBytes += entry.Size() + } + + for _, entry := range s.currs { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + for _, entry := range s.matching { + if entry != nil { + sizeInBytes += entry.Size() + } + } + + sizeInBytes += len(s.matchingIdxs) * size.SizeOfInt + + return sizeInBytes +} + +func (s *DisjunctionSliceSearcher) computeQueryNorm() { + // first calculate sum of squared weights + sumOfSquaredWeights := 0.0 + for _, searcher := range s.searchers { + sumOfSquaredWeights += searcher.Weight() + } + // now compute query norm from this + s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights) + // finally tell all the downstream searchers the norm + for _, searcher := range s.searchers { + searcher.SetQueryNorm(s.queryNorm) + } +} + +func (s *DisjunctionSliceSearcher) initSearchers(ctx *search.SearchContext) error { + var err error + // get all searchers pointing at their first match + for i, searcher := range s.searchers { + if s.currs[i] != nil { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) + if err != nil { + return err + } + } + + err = s.updateMatches() + if err != nil { + return err + } + + s.initialized = true + return nil +} + +func (s *DisjunctionSliceSearcher) updateMatches() error { + matching := s.matching[:0] + matchingIdxs := s.matchingIdxs[:0] + + for i := 0; i < len(s.currs); i++ { + curr := s.currs[i] + if curr == nil { + continue + } + + if len(matching) > 0 { + cmp := curr.IndexInternalID.Compare(matching[0].IndexInternalID) + if cmp > 0 { + continue + } + + if cmp < 0 { + matching = matching[:0] + matchingIdxs = matchingIdxs[:0] + } + } + + matching = append(matching, curr) + matchingIdxs = append(matchingIdxs, i) + } + + s.matching = matching + s.matchingIdxs = matchingIdxs + + return nil +} + +func (s *DisjunctionSliceSearcher) Weight() float64 { + var rv float64 + for _, searcher := range s.searchers { + rv += searcher.Weight() + } + return rv +} + +func (s *DisjunctionSliceSearcher) SetQueryNorm(qnorm float64) { + for _, searcher := range s.searchers { + searcher.SetQueryNorm(qnorm) + } +} + +func (s *DisjunctionSliceSearcher) Next(ctx *search.SearchContext) ( + *search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + var err error + var rv *search.DocumentMatch + + found := false + for !found && len(s.matching) > 0 { + if len(s.matching) >= s.min { + found = true + // score this match + rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers) + } + + // invoke next on all the matching searchers + for _, i := range s.matchingIdxs { + searcher := s.searchers[i] + if s.currs[i] != rv { + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Next(ctx) + if err != nil { + return nil, err + } + } + + err = s.updateMatches() + if err != nil { + return nil, err + } + } + return rv, nil +} + +func (s *DisjunctionSliceSearcher) Advance(ctx *search.SearchContext, + ID index.IndexInternalID) (*search.DocumentMatch, error) { + if !s.initialized { + err := s.initSearchers(ctx) + if err != nil { + return nil, err + } + } + // get all searchers pointing at their first match + var err error + for i, searcher := range s.searchers { + if s.currs[i] != nil { + if s.currs[i].IndexInternalID.Compare(ID) >= 0 { + continue + } + ctx.DocumentMatchPool.Put(s.currs[i]) + } + s.currs[i], err = searcher.Advance(ctx, ID) + if err != nil { + return nil, err + } + } + + err = s.updateMatches() + if err != nil { + return nil, err + } + + return s.Next(ctx) +} + +func (s *DisjunctionSliceSearcher) Count() uint64 { + // for now return a worst case + var sum uint64 + for _, searcher := range s.searchers { + sum += searcher.Count() + } + return sum +} + +func (s *DisjunctionSliceSearcher) Close() (rv error) { + for _, searcher := range s.searchers { + err := searcher.Close() + if err != nil && rv == nil { + rv = err + } + } + return rv +} + +func (s *DisjunctionSliceSearcher) Min() int { + return s.min +} + +func (s *DisjunctionSliceSearcher) DocumentMatchPoolSize() int { + rv := len(s.currs) + for _, s := range s.searchers { + rv += s.DocumentMatchPoolSize() + } + return rv +} + +// a disjunction searcher implements the index.Optimizable interface +// but only activates on an edge case where the disjunction is a +// wrapper around a single Optimizable child searcher +func (s *DisjunctionSliceSearcher) Optimize(kind string, octx index.OptimizableContext) ( + index.OptimizableContext, error) { + if len(s.searchers) == 1 { + o, ok := s.searchers[0].(index.Optimizable) + if ok { + return o.Optimize(kind, octx) + } + } + + return octx, nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go index 06351b4a..3b258a58 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_docid.go @@ -15,11 +15,21 @@ package searcher import ( + "reflect" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search/scorer" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeDocIDSearcher int + +func init() { + var ds DocIDSearcher + reflectStaticSizeDocIDSearcher = int(reflect.TypeOf(ds).Size()) +} + // DocIDSearcher returns documents matching a predefined set of identifiers. type DocIDSearcher struct { reader index.DocIDReader @@ -42,6 +52,12 @@ func NewDocIDSearcher(indexReader index.IndexReader, ids []string, boost float64 }, nil } +func (s *DocIDSearcher) Size() int { + return reflectStaticSizeDocIDSearcher + size.SizeOfPtr + + s.reader.Size() + + s.scorer.Size() +} + func (s *DocIDSearcher) Count() uint64 { return uint64(s.count) } diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go index 219f2ee7..7c95fb41 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_filter.go @@ -15,10 +15,20 @@ package searcher import ( + "reflect" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeFilteringSearcher int + +func init() { + var fs FilteringSearcher + reflectStaticSizeFilteringSearcher = int(reflect.TypeOf(fs).Size()) +} + // FilterFunc defines a function which can filter documents // returning true means keep the document // returning false means do not keep the document @@ -38,6 +48,11 @@ func NewFilteringSearcher(s search.Searcher, filter FilterFunc) *FilteringSearch } } +func (f *FilteringSearcher) Size() int { + return reflectStaticSizeFilteringSearcher + size.SizeOfPtr + + f.child.Size() +} + func (f *FilteringSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) { next, err := f.child.Next(ctx) for next != nil && err == nil { diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go index 90abaa0a..b812f484 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_fuzzy.go @@ -15,13 +15,22 @@ package searcher import ( + "fmt" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" ) +var MaxFuzziness = 2 + func NewFuzzySearcher(indexReader index.IndexReader, term string, prefix, fuzziness int, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) { + + if fuzziness > MaxFuzziness { + return nil, fmt.Errorf("fuzziness exceeds max (%d)", MaxFuzziness) + } + // Note: we don't byte slice the term for a prefix because of runes. prefixTerm := "" for i, r := range term { @@ -31,7 +40,6 @@ func NewFuzzySearcher(indexReader index.IndexReader, term string, break } } - candidateTerms, err := findFuzzyCandidateTerms(indexReader, term, fuzziness, field, prefixTerm) if err != nil { @@ -49,6 +57,26 @@ func findFuzzyCandidateTerms(indexReader index.IndexReader, term string, if len(prefixTerm) > 0 { fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm)) } else { + // in case of advanced reader implementations directly call + // the levenshtein automaton based iterator to collect the + // candidate terms + if ir, ok := indexReader.(index.IndexReaderFuzzy); ok { + fieldDict, err = ir.FieldDictFuzzy(field, []byte(term), fuzziness) + if err != nil { + return rv, err + } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + rv = append(rv, tfd.Term) + tfd, err = fieldDict.Next() + } + return rv, err + } fieldDict, err = indexReader.FieldDict(field) } defer func() { @@ -58,9 +86,12 @@ func findFuzzyCandidateTerms(indexReader index.IndexReader, term string, }() // enumerate terms and check levenshtein distance + var reuse []int tfd, err := fieldDict.Next() for err == nil && tfd != nil { - ld, exceeded := search.LevenshteinDistanceMax(term, tfd.Term, fuzziness) + var ld int + var exceeded bool + ld, exceeded, reuse = search.LevenshteinDistanceMaxReuseSlice(term, tfd.Term, fuzziness, reuse) if !exceeded && ld <= fuzziness { rv = append(rv, tfd.Term) if tooManyClauses(len(rv)) { diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go index c9d347fa..fd559766 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_geopointdistance.go @@ -24,10 +24,12 @@ import ( func NewGeoPointDistanceSearcher(indexReader index.IndexReader, centerLon, centerLat, dist float64, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) { - // compute bounding box containing the circle - topLeftLon, topLeftLat, bottomRightLon, bottomRightLat := - geo.ComputeBoundingBox(centerLon, centerLat, dist) + topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err := + geo.RectFromPointDistance(centerLon, centerLat, dist) + if err != nil { + return nil, err + } // build a searcher for the box boxSearcher, err := boxSearcher(indexReader, diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go index 822db2ea..bb664012 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_match_all.go @@ -15,11 +15,21 @@ package searcher import ( + "reflect" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search/scorer" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeMatchAllSearcher int + +func init() { + var mas MatchAllSearcher + reflectStaticSizeMatchAllSearcher = int(reflect.TypeOf(mas).Size()) +} + type MatchAllSearcher struct { indexReader index.IndexReader reader index.DocIDReader @@ -46,6 +56,12 @@ func NewMatchAllSearcher(indexReader index.IndexReader, boost float64, options s }, nil } +func (s *MatchAllSearcher) Size() int { + return reflectStaticSizeMatchAllSearcher + size.SizeOfPtr + + s.reader.Size() + + s.scorer.Size() +} + func (s *MatchAllSearcher) Count() uint64 { return s.count } diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go index 94759671..a345e17f 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_match_none.go @@ -15,10 +15,20 @@ package searcher import ( + "reflect" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeMatchNoneSearcher int + +func init() { + var mns MatchNoneSearcher + reflectStaticSizeMatchNoneSearcher = int(reflect.TypeOf(mns).Size()) +} + type MatchNoneSearcher struct { indexReader index.IndexReader } @@ -29,6 +39,10 @@ func NewMatchNoneSearcher(indexReader index.IndexReader) (*MatchNoneSearcher, er }, nil } +func (s *MatchNoneSearcher) Size() int { + return reflectStaticSizeMatchNoneSearcher + size.SizeOfPtr +} + func (s *MatchNoneSearcher) Count() uint64 { return uint64(0) } diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go index 224cc9e2..1eae7a5e 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_numeric_range.go @@ -17,6 +17,7 @@ package searcher import ( "bytes" "math" + "sort" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/numeric" @@ -55,6 +56,17 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, // FIXME hard-coded precision, should match field declaration termRanges := splitInt64Range(minInt64, maxInt64, 4) terms := termRanges.Enumerate() + if len(terms) < 1 { + // cannot return MatchNoneSearcher because of interaction with + // commit f391b991c20f02681bacd197afc6d8aed444e132 + return NewMultiTermSearcherBytes(indexReader, terms, field, boost, options, + true) + } + var err error + terms, err = filterCandidateTerms(indexReader, terms, field) + if err != nil { + return nil, err + } if tooManyClauses(len(terms)) { return nil, tooManyClausesErr() } @@ -63,6 +75,51 @@ func NewNumericRangeSearcher(indexReader index.IndexReader, true) } +func filterCandidateTerms(indexReader index.IndexReader, + terms [][]byte, field string) (rv [][]byte, err error) { + + if ir, ok := indexReader.(index.IndexReaderOnly); ok { + fieldDict, err := ir.FieldDictOnly(field, terms, false) + if err != nil { + return nil, err + } + // enumerate the terms (no need to check them again) + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + rv = append(rv, []byte(tfd.Term)) + tfd, err = fieldDict.Next() + } + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + + return rv, err + } + + fieldDict, err := indexReader.FieldDictRange(field, terms[0], terms[len(terms)-1]) + if err != nil { + return nil, err + } + + // enumerate the terms and check against list of terms + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + termBytes := []byte(tfd.Term) + i := sort.Search(len(terms), func(i int) bool { return bytes.Compare(terms[i], termBytes) >= 0 }) + if i < len(terms) && bytes.Compare(terms[i], termBytes) == 0 { + rv = append(rv, terms[i]) + } + terms = terms[i:] + tfd, err = fieldDict.Next() + } + + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + + return rv, err +} + type termRange struct { startTerm []byte endTerm []byte diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go index e3fa0895..3711da06 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go @@ -17,21 +17,52 @@ package searcher import ( "fmt" "math" + "reflect" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizePhraseSearcher int + +func init() { + var ps PhraseSearcher + reflectStaticSizePhraseSearcher = int(reflect.TypeOf(ps).Size()) +} + type PhraseSearcher struct { - indexReader index.IndexReader mustSearcher *ConjunctionSearcher queryNorm float64 currMust *search.DocumentMatch - slop int terms [][]string + path phrasePath + paths []phrasePath + locations []search.Location initialized bool } +func (s *PhraseSearcher) Size() int { + sizeInBytes := reflectStaticSizePhraseSearcher + size.SizeOfPtr + + if s.mustSearcher != nil { + sizeInBytes += s.mustSearcher.Size() + } + + if s.currMust != nil { + sizeInBytes += s.currMust.Size() + } + + for _, entry := range s.terms { + sizeInBytes += size.SizeOfSlice + for _, entry1 := range entry { + sizeInBytes += size.SizeOfString + len(entry1) + } + } + + return sizeInBytes +} + func NewPhraseSearcher(indexReader index.IndexReader, terms []string, field string, options search.SearcherOptions) (*PhraseSearcher, error) { // turn flat terms []string into [][]string mterms := make([][]string, len(terms)) @@ -96,7 +127,6 @@ func NewMultiPhraseSearcher(indexReader index.IndexReader, terms [][]string, fie // build our searcher rv := PhraseSearcher{ - indexReader: indexReader, mustSearcher: mustSearcher, terms: terms, } @@ -133,6 +163,9 @@ func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error { var err error if s.mustSearcher != nil { + if s.currMust != nil { + ctx.DocumentMatchPool.Put(s.currMust) + } s.currMust, err = s.mustSearcher.Next(ctx) if err != nil { return err @@ -182,43 +215,59 @@ func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, // also satisfies the phase constraints. if so, it returns a DocumentMatch // for this document, otherwise nil func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch { - rvftlm := make(search.FieldTermLocationMap, 0) - freq := 0 + s.locations = s.currMust.Complete(s.locations) + + locations := s.currMust.Locations + s.currMust.Locations = nil + + ftls := s.currMust.FieldTermLocations + // typically we would expect there to only actually be results in // one field, but we allow for this to not be the case // but, we note that phrase constraints can only be satisfied within // a single field, so we can check them each independently - for field, tlm := range s.currMust.Locations { - - f, rvtlm := s.checkCurrMustMatchField(ctx, tlm) - if f > 0 { - freq += f - rvftlm[field] = rvtlm - } + for field, tlm := range locations { + ftls = s.checkCurrMustMatchField(ctx, field, tlm, ftls) } - if freq > 0 { + if len(ftls) > 0 { // return match rv := s.currMust - rv.Locations = rvftlm + s.currMust = nil + rv.FieldTermLocations = ftls return rv } return nil } -// checkCurrMustMatchField is soley concerned with determining if one particular -// field within the currMust DocumentMatch Locations satisfies the phase -// constraints (possibly more than once). if so, the number of times it was -// satisfied, and these locations are returned. otherwise 0 and either -// a nil or empty TermLocationMap -func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, tlm search.TermLocationMap) (int, search.TermLocationMap) { - paths := findPhrasePaths(0, nil, s.terms, tlm, nil, 0) - rv := make(search.TermLocationMap, len(s.terms)) - for _, p := range paths { - p.MergeInto(rv) +// checkCurrMustMatchField is soley concerned with determining if one +// particular field within the currMust DocumentMatch Locations +// satisfies the phase constraints (possibly more than once). if so, +// the matching field term locations are appended to the provided +// slice +func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, + field string, tlm search.TermLocationMap, + ftls []search.FieldTermLocation) []search.FieldTermLocation { + if s.path == nil { + s.path = make(phrasePath, 0, len(s.terms)) } - return len(paths), rv + s.paths = findPhrasePaths(0, nil, s.terms, tlm, s.path[:0], 0, s.paths[:0]) + for _, p := range s.paths { + for _, pp := range p { + ftls = append(ftls, search.FieldTermLocation{ + Field: field, + Term: pp.term, + Location: search.Location{ + Pos: pp.loc.Pos, + Start: pp.loc.Start, + End: pp.loc.End, + ArrayPositions: pp.loc.ArrayPositions, + }, + }) + } + } + return ftls } type phrasePart struct { @@ -226,7 +275,11 @@ type phrasePart struct { loc *search.Location } -type phrasePath []*phrasePart +func (p *phrasePart) String() string { + return fmt.Sprintf("[%s %v]", p.term, p.loc) +} + +type phrasePath []phrasePart func (p phrasePath) MergeInto(in search.TermLocationMap) { for _, pp := range p { @@ -234,24 +287,51 @@ func (p phrasePath) MergeInto(in search.TermLocationMap) { } } -// findPhrasePaths is a function to identify phase matches from a set of known -// term locations. the implementation is recursive, so care must be taken -// with arguments and return values. +func (p phrasePath) String() string { + rv := "[" + for i, pp := range p { + if i > 0 { + rv += ", " + } + rv += pp.String() + } + rv += "]" + return rv +} + +// findPhrasePaths is a function to identify phase matches from a set +// of known term locations. it recursive so care must be taken with +// arguments and return values. // -// prev - the previous location, nil on first invocation -// phraseTerms - slice containing the phrase terms themselves +// prevPos - the previous location, 0 on first invocation +// ap - array positions of the first candidate phrase part to +// which further recursive phrase parts must match, +// nil on initial invocation or when there are no array positions +// phraseTerms - slice containing the phrase terms, // may contain empty string as placeholder (don't care) // tlm - the Term Location Map containing all relevant term locations -// offset - the offset from the previous that this next term must match // p - the current path being explored (appended to in recursive calls) // this is the primary state being built during the traversal +// remainingSlop - amount of sloppiness that's allowed, which is the +// sum of the editDistances from each matching phrase part, +// where 0 means no sloppiness allowed (all editDistances must be 0), +// decremented during recursion +// rv - the final result being appended to by all the recursive calls // // returns slice of paths, or nil if invocation did not find any successul paths -func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string, tlm search.TermLocationMap, p phrasePath, remainingSlop int) []phrasePath { - +func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string, + tlm search.TermLocationMap, p phrasePath, remainingSlop int, rv []phrasePath) []phrasePath { // no more terms if len(phraseTerms) < 1 { - return []phrasePath{p} + // snapshot or copy the recursively built phrasePath p and + // append it to the rv, also optimizing by checking if next + // phrasePath item in the rv (which we're about to overwrite) + // is available for reuse + var pcopy phrasePath + if len(rv) < cap(rv) { + pcopy = rv[:len(rv)+1][len(rv)][:0] + } + return append(rv, append(pcopy, p...)) } car := phraseTerms[0] @@ -264,13 +344,13 @@ func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]s // if prevPos was 0, don't set it to 1 (as thats not a real abs pos) nextPos = 0 // don't advance nextPos if prevPos was 0 } - return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop) + return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop, rv) } - var rv []phrasePath // locations for this term for _, carTerm := range car { locations := tlm[carTerm] + LOCATIONS_LOOP: for _, loc := range locations { if prevPos != 0 && !loc.ArrayPositions.Equals(ap) { // if the array positions are wrong, can't match, try next location @@ -283,11 +363,18 @@ func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]s dist = editDistance(prevPos+1, loc.Pos) } - // if enough slop reamining, continue recursively + // if enough slop remaining, continue recursively if prevPos == 0 || (remainingSlop-dist) >= 0 { + // skip if we've already used this term+loc already + for _, ppart := range p { + if ppart.term == carTerm && ppart.loc == loc { + continue LOCATIONS_LOOP + } + } + // this location works, add it to the path (but not for empty term) - px := append(p, &phrasePart{term: carTerm, loc: loc}) - rv = append(rv, findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist)...) + px := append(p, phrasePart{term: carTerm, loc: loc}) + rv = findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist, rv) } } } @@ -309,6 +396,15 @@ func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexIntern return nil, err } } + if s.currMust != nil { + if s.currMust.IndexInternalID.Compare(ID) >= 0 { + return s.Next(ctx) + } + ctx.DocumentMatchPool.Put(s.currMust) + } + if s.currMust == nil { + return nil, nil + } var err error s.currMust, err = s.mustSearcher.Advance(ctx, ID) if err != nil { diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go index b7cf520a..ad417a05 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_regexp.go @@ -29,19 +29,40 @@ import ( func NewRegexpSearcher(indexReader index.IndexReader, pattern *regexp.Regexp, field string, boost float64, options search.SearcherOptions) ( search.Searcher, error) { - - prefixTerm, complete := pattern.LiteralPrefix() var candidateTerms []string - if complete { - // there is no pattern - candidateTerms = []string{prefixTerm} - } else { - var err error - candidateTerms, err = findRegexpCandidateTerms(indexReader, pattern, field, - prefixTerm) + if ir, ok := indexReader.(index.IndexReaderRegexp); ok { + fieldDict, err := ir.FieldDictRegexp(field, []byte(pattern.String())) if err != nil { return nil, err } + defer func() { + if cerr := fieldDict.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + // enumerate the terms and check against regexp + tfd, err := fieldDict.Next() + for err == nil && tfd != nil { + candidateTerms = append(candidateTerms, tfd.Term) + tfd, err = fieldDict.Next() + } + if err != nil { + return nil, err + } + } else { + prefixTerm, complete := pattern.LiteralPrefix() + if complete { + // there is no pattern + candidateTerms = []string{prefixTerm} + } else { + var err error + candidateTerms, err = findRegexpCandidateTerms(indexReader, pattern, field, + prefixTerm) + if err != nil { + return nil, err + } + } } return NewMultiTermSearcher(indexReader, candidateTerms, field, boost, diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go index 6fae6ae5..4fee58bb 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_term.go @@ -15,11 +15,21 @@ package searcher import ( + "reflect" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search/scorer" + "github.com/blevesearch/bleve/size" ) +var reflectStaticSizeTermSearcher int + +func init() { + var ts TermSearcher + reflectStaticSizeTermSearcher = int(reflect.TypeOf(ts).Size()) +} + type TermSearcher struct { indexReader index.IndexReader reader index.TermFieldReader @@ -63,6 +73,13 @@ func NewTermSearcherBytes(indexReader index.IndexReader, term []byte, field stri }, nil } +func (s *TermSearcher) Size() int { + return reflectStaticSizeTermSearcher + size.SizeOfPtr + + s.reader.Size() + + s.tfd.Size() + + s.scorer.Size() +} + func (s *TermSearcher) Count() uint64 { return s.reader.Count() } @@ -120,3 +137,13 @@ func (s *TermSearcher) Min() int { func (s *TermSearcher) DocumentMatchPoolSize() int { return 1 } + +func (s *TermSearcher) Optimize(kind string, octx index.OptimizableContext) ( + index.OptimizableContext, error) { + o, ok := s.reader.(index.Optimizable) + if ok { + return o.Optimize(kind, octx) + } + + return octx, nil +} diff --git a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go b/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go index 24170c71..267c681b 100644 --- a/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go +++ b/vendor/github.com/blevesearch/bleve/search/searcher/search_term_range.go @@ -64,6 +64,10 @@ func NewTermRangeSearcher(indexReader index.IndexReader, if !*inclusiveMin && min != nil && string(min) == terms[0] { terms = terms[1:] + // check again, as we might have removed only entry + if len(terms) < 1 { + return NewMatchNoneSearcher(indexReader) + } } // if our term list included the max, it would be the last item diff --git a/vendor/github.com/blevesearch/bleve/search/sort.go b/vendor/github.com/blevesearch/bleve/search/sort.go index 1987718b..6afc9789 100644 --- a/vendor/github.com/blevesearch/bleve/search/sort.go +++ b/vendor/github.com/blevesearch/bleve/search/sort.go @@ -67,8 +67,8 @@ func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { rvd := &SortGeoDistance{ Field: field, Desc: descending, - lon: lon, - lat: lat, + Lon: lon, + Lat: lat, unitMult: 1.0, } if distUnit, ok := input["unit"].(string); ok { @@ -251,23 +251,21 @@ func (so SortOrder) Compare(cachedScoring, cachedDesc []bool, i, j *DocumentMatc } func (so SortOrder) RequiresScore() bool { - rv := false for _, soi := range so { if soi.RequiresScoring() { - rv = true + return true } } - return rv + return false } func (so SortOrder) RequiresDocID() bool { - rv := false for _, soi := range so { if soi.RequiresDocID() { - rv = true + return true } } - return rv + return false } func (so SortOrder) RequiredFields() []string { @@ -279,7 +277,7 @@ func (so SortOrder) RequiredFields() []string { } func (so SortOrder) CacheIsScore() []bool { - var rv []bool + rv := make([]bool, 0, len(so)) for _, soi := range so { rv = append(rv, soi.RequiresScoring()) } @@ -287,7 +285,7 @@ func (so SortOrder) CacheIsScore() []bool { } func (so SortOrder) CacheDescending() []bool { - var rv []bool + rv := make([]bool, 0, len(so)) for _, soi := range so { rv = append(rv, soi.Descending()) } @@ -486,8 +484,7 @@ func (s *SortField) MarshalJSON() ([]byte, error) { } func (s *SortField) Copy() SearchSort { - var rv SortField - rv = *s + rv := *s return &rv } @@ -499,7 +496,6 @@ type SortDocID struct { // UpdateVisitor is a no-op for SortDocID as it's value // is not dependent on any field terms func (s *SortDocID) UpdateVisitor(field string, term []byte) { - } // Value returns the sort value of the DocumentMatch @@ -529,8 +525,7 @@ func (s *SortDocID) MarshalJSON() ([]byte, error) { } func (s *SortDocID) Copy() SearchSort { - var rv SortDocID - rv = *s + rv := *s return &rv } @@ -542,7 +537,6 @@ type SortScore struct { // UpdateVisitor is a no-op for SortScore as it's value // is not dependent on any field terms func (s *SortScore) UpdateVisitor(field string, term []byte) { - } // Value returns the sort value of the DocumentMatch @@ -572,8 +566,7 @@ func (s *SortScore) MarshalJSON() ([]byte, error) { } func (s *SortScore) Copy() SearchSort { - var rv SortScore - rv = *s + rv := *s return &rv } @@ -583,13 +576,12 @@ var maxDistance = string(numeric.MustNewPrefixCodedInt64(math.MaxInt64, 0)) // their distance from the specified point. func NewSortGeoDistance(field, unit string, lon, lat float64, desc bool) ( *SortGeoDistance, error) { - rv := &SortGeoDistance{ Field: field, Desc: desc, Unit: unit, - lon: lon, - lat: lat, + Lon: lon, + Lat: lat, } var err error rv.unitMult, err = geo.ParseDistanceUnit(unit) @@ -608,8 +600,8 @@ type SortGeoDistance struct { Desc bool Unit string values []string - lon float64 - lat float64 + Lon float64 + Lat float64 unitMult float64 } @@ -640,7 +632,7 @@ func (s *SortGeoDistance) Value(i *DocumentMatch) string { docLon := geo.MortonUnhashLon(uint64(i64)) docLat := geo.MortonUnhashLat(uint64(i64)) - dist := geo.Haversin(s.lon, s.lat, docLon, docLat) + dist := geo.Haversin(s.Lon, s.Lat, docLon, docLat) // dist is returned in km, so convert to m dist *= 1000 if s.unitMult != 0 { @@ -690,8 +682,8 @@ func (s *SortGeoDistance) MarshalJSON() ([]byte, error) { "by": "geo_distance", "field": s.Field, "location": map[string]interface{}{ - "lon": s.lon, - "lat": s.lat, + "lon": s.Lon, + "lat": s.Lat, }, } if s.Unit != "" { @@ -705,7 +697,6 @@ func (s *SortGeoDistance) MarshalJSON() ([]byte, error) { } func (s *SortGeoDistance) Copy() SearchSort { - var rv SortGeoDistance - rv = *s + rv := *s return &rv } diff --git a/vendor/github.com/blevesearch/bleve/search/util.go b/vendor/github.com/blevesearch/bleve/search/util.go index 83212af1..19dd5d68 100644 --- a/vendor/github.com/blevesearch/bleve/search/util.go +++ b/vendor/github.com/blevesearch/bleve/search/util.go @@ -40,3 +40,30 @@ func MergeTermLocationMaps(rv, other TermLocationMap) TermLocationMap { } return rv } + +func MergeFieldTermLocations(dest []FieldTermLocation, matches []*DocumentMatch) []FieldTermLocation { + n := len(dest) + for _, dm := range matches { + n += len(dm.FieldTermLocations) + } + if cap(dest) < n { + dest = append(make([]FieldTermLocation, 0, n), dest...) + } + + for _, dm := range matches { + for _, ftl := range dm.FieldTermLocations { + dest = append(dest, FieldTermLocation{ + Field: ftl.Field, + Term: ftl.Term, + Location: Location{ + Pos: ftl.Location.Pos, + Start: ftl.Location.Start, + End: ftl.Location.End, + ArrayPositions: append(ArrayPositions(nil), ftl.Location.ArrayPositions...), + }, + }) + } + } + + return dest +} diff --git a/vendor/github.com/blevesearch/bleve/size/sizes.go b/vendor/github.com/blevesearch/bleve/size/sizes.go new file mode 100644 index 00000000..0990bf86 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/size/sizes.go @@ -0,0 +1,59 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package size + +import ( + "reflect" +) + +func init() { + var b bool + SizeOfBool = int(reflect.TypeOf(b).Size()) + var f32 float32 + SizeOfFloat32 = int(reflect.TypeOf(f32).Size()) + var f64 float64 + SizeOfFloat64 = int(reflect.TypeOf(f64).Size()) + var i int + SizeOfInt = int(reflect.TypeOf(i).Size()) + var m map[int]int + SizeOfMap = int(reflect.TypeOf(m).Size()) + var ptr *int + SizeOfPtr = int(reflect.TypeOf(ptr).Size()) + var slice []int + SizeOfSlice = int(reflect.TypeOf(slice).Size()) + var str string + SizeOfString = int(reflect.TypeOf(str).Size()) + var u8 uint8 + SizeOfUint8 = int(reflect.TypeOf(u8).Size()) + var u16 uint16 + SizeOfUint16 = int(reflect.TypeOf(u16).Size()) + var u32 uint32 + SizeOfUint32 = int(reflect.TypeOf(u32).Size()) + var u64 uint64 + SizeOfUint64 = int(reflect.TypeOf(u64).Size()) +} + +var SizeOfBool int +var SizeOfFloat32 int +var SizeOfFloat64 int +var SizeOfInt int +var SizeOfMap int +var SizeOfPtr int +var SizeOfSlice int +var SizeOfString int +var SizeOfUint8 int +var SizeOfUint16 int +var SizeOfUint32 int +var SizeOfUint64 int diff --git a/vendor/github.com/boltdb/bolt/README.md b/vendor/github.com/boltdb/bolt/README.md index d7f80e97..fc2a1e04 100644 --- a/vendor/github.com/boltdb/bolt/README.md +++ b/vendor/github.com/boltdb/bolt/README.md @@ -21,6 +21,25 @@ consistency and thread safety. Bolt is currently used in high-load production environments serving databases as large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed services every day. +## A message from the author + +> The original goal of Bolt was to provide a simple pure Go key/value store and to +> not bloat the code with extraneous features. To that end, the project has been +> a success. However, this limited scope also means that the project is complete. +> +> Maintaining an open source database requires an immense amount of time and energy. +> Changes to the code can have unintended and sometimes catastrophic effects so +> even simple changes require hours and hours of careful testing and validation. +> +> Unfortunately I no longer have the time or energy to continue this work. Bolt is +> in a stable state and has years of successful production use. As such, I feel that +> leaving it in its current state is the most prudent course of action. +> +> If you are interested in using a more featureful version of Bolt, I suggest that +> you look at the CoreOS fork called [bbolt](https://github.com/coreos/bbolt). + +- Ben Johnson ([@benbjohnson](https://twitter.com/benbjohnson)) + ## Table of Contents - [Getting Started](#getting-started) @@ -911,5 +930,6 @@ Below is a list of public, open source projects that use Bolt: * [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. * [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies * [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB +* [Ponzu CMS](https://ponzu-cms.org) - Headless CMS + automatic JSON API with auto-HTTPS, HTTP/2 Server Push, and flexible server framework. If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/boltdb/bolt/db.go b/vendor/github.com/boltdb/bolt/db.go index f352ff14..cc3596fa 100644 --- a/vendor/github.com/boltdb/bolt/db.go +++ b/vendor/github.com/boltdb/bolt/db.go @@ -737,9 +737,7 @@ retry: // pass success, or bolt internal errors, to all callers for _, c := range b.calls { - if c.err != nil { - c.err <- err - } + c.err <- err } break retry } diff --git a/vendor/github.com/boltdb/bolt/tx.go b/vendor/github.com/boltdb/bolt/tx.go index 6700308a..bbb60ac9 100644 --- a/vendor/github.com/boltdb/bolt/tx.go +++ b/vendor/github.com/boltdb/bolt/tx.go @@ -291,7 +291,9 @@ func (tx *Tx) close() { } // Copy writes the entire database to a writer. -// This function exists for backwards compatibility. Use WriteTo() instead. +// This function exists for backwards compatibility. +// +// Deprecated; Use WriteTo() instead. func (tx *Tx) Copy(w io.Writer) error { _, err := tx.WriteTo(w) return err diff --git a/vendor/github.com/couchbase/vellum/CONTRIBUTING.md b/vendor/github.com/couchbase/vellum/CONTRIBUTING.md new file mode 100644 index 00000000..b85ec82b --- /dev/null +++ b/vendor/github.com/couchbase/vellum/CONTRIBUTING.md @@ -0,0 +1,16 @@ +# Contributing to Vellum + +We look forward to your contributions, but ask that you first review these guidelines. + +### Sign the CLA + +As Vellum is a Couchbase project we require contributors accept the [Couchbase Contributor License Agreement](http://review.couchbase.org/static/individual_agreement.html). To sign this agreement log into the Couchbase [code review tool](http://review.couchbase.org/). The Vellum project does not use this code review tool but it is still used to track acceptance of the contributor license agreements. + +### Submitting a Pull Request + +All types of contributions are welcome, but please keep the following in mind: + +- If you're planning a large change, you should really discuss it in a github issue first. This helps avoid duplicate effort and spending time on something that may not be merged. +- Existing tests should continue to pass, new tests for the contribution are nice to have. +- All code should have gone through `go fmt` +- All code should pass `go vet` diff --git a/vendor/github.com/couchbase/vellum/LICENSE b/vendor/github.com/couchbase/vellum/LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/couchbase/vellum/README.md b/vendor/github.com/couchbase/vellum/README.md new file mode 100644 index 00000000..0c0759a9 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/README.md @@ -0,0 +1,168 @@ +# ![vellum](docs/logo.png) vellum + +[![Build Status](https://travis-ci.org/couchbase/vellum.svg?branch=master)](https://travis-ci.org/couchbase/vellum) +[![Coverage Status](https://coveralls.io/repos/github/couchbase/vellum/badge.svg?branch=master)](https://coveralls.io/github/couchbase/vellum?branch=master) +[![GoDoc](https://godoc.org/github.com/couchbase/vellum?status.svg)](https://godoc.org/github.com/couchbase/vellum) +[![Go Report Card](https://goreportcard.com/badge/github.com/couchbase/vellum)](https://goreportcard.com/report/github.com/couchbase/vellum) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +A Go library implementing an FST (finite state transducer) capable of: + - mapping between keys ([]byte) and a value (uint64) + - enumerating keys in lexicographic order + +Some additional goals of this implementation: + - bounded memory use while building the FST + - streaming out FST data while building + - mmap FST runtime to support very large FTSs (optional) + +## Usage + +### Building an FST + +To build an FST, create a new builder using the `New()` method. This method takes an `io.Writer` as an argument. As the FST is being built, data will be streamed to the writer as soon as possible. With this builder you **MUST** insert keys in lexicographic order. Inserting keys out of order will result in an error. After inserting the last key into the builder, you **MUST** call `Close()` on the builder. This will flush all remaining data to the underlying writer. + +In memory: +```go + var buf bytes.Buffer + builder, err := vellum.New(&buf, nil) + if err != nil { + log.Fatal(err) + } +``` + +To disk: +```go + f, err := os.Create("/tmp/vellum.fst") + if err != nil { + log.Fatal(err) + } + builder, err := vellum.New(f, nil) + if err != nil { + log.Fatal(err) + } +``` + +**MUST** insert keys in lexicographic order: +```go +err = builder.Insert([]byte("cat"), 1) +if err != nil { + log.Fatal(err) +} + +err = builder.Insert([]byte("dog"), 2) +if err != nil { + log.Fatal(err) +} + +err = builder.Insert([]byte("fish"), 3) +if err != nil { + log.Fatal(err) +} + +err = builder.Close() +if err != nil { + log.Fatal(err) +} +``` + +### Using an FST + +After closing the builder, the data can be used to instantiate an FST. If the data was written to disk, you can use the `Open()` method to mmap the file. If the data is already in memory, or you wish to load/mmap the data yourself, you can instantiate the FST with the `Load()` method. + +Load in memory: +```go + fst, err := vellum.Load(buf.Bytes()) + if err != nil { + log.Fatal(err) + } +``` + +Open from disk: +```go + fst, err := vellum.Open("/tmp/vellum.fst") + if err != nil { + log.Fatal(err) + } +``` + +Get key/value: +```go + val, exists, err = fst.Get([]byte("dog")) + if err != nil { + log.Fatal(err) + } + if exists { + fmt.Printf("contains dog with val: %d\n", val) + } else { + fmt.Printf("does not contain dog") + } +``` + +Iterate key/values: +```go + itr, err := fst.Iterator(startKeyInclusive, endKeyExclusive) + for err == nil { + key, val := itr.Current() + fmt.Printf("contains key: %s val: %d", key, val) + err = itr.Next() + } + if err != nil { + log.Fatal(err) + } +``` + +### How does the FST get built? + +A full example of the implementation is beyond the scope of this README, but let's consider a small example where we want to insert 3 key/value pairs. + +First we insert "are" with the value 4. + +![step1](docs/demo1.png) + +Next, we insert "ate" with the value 2. + +![step2](docs/demo2.png) + +Notice how the values associated with the transitions were adjusted so that by summing them while traversing we still get the expected value. + +At this point, we see that state 5 looks like state 3, and state 4 looks like state 2. But, we cannot yet combine them because future inserts could change this. + +Now, we insert "see" with value 3. Once it has been added, we now know that states 5 and 4 can longer change. Since they are identical to 3 and 2, we replace them. + +![step3](docs/demo3.png) + +Again, we see that states 7 and 8 appear to be identical to 2 and 3. + +Having inserted our last key, we call `Close()` on the builder. + +![step4](docs/demo4.png) + +Now, states 7 and 8 can safely be replaced with 2 and 3. + +For additional information, see the references at the bottom of this document. + +### What does the serialized format look like? + +We've broken out a separate document on the [vellum disk format v1](docs/format.md). + +### What if I want to use this on a system that doesn't have mmap? + +The mmap library itself is guarded with system/architecture build tags, but we've also added an additional build tag in vellum. If you'd like to Open() a file based representation of an FST, but not use mmap, you can build the library with the `nommap` build tag. NOTE: if you do this, the entire FST will be read into memory. + +### Can I use this with Unicode strings? + +Yes, however this implementation is only aware of the byte representation you choose. In order to find matches, you must work with some canonical byte representation of the string. In the future, some encoding-aware traversals may be possible on top of the lower-level byte transitions. + +### How did this library come to be? + +In my work on the [Bleve](https://github.com/blevesearch/bleve) project I became aware of the power of the FST for many search-related tasks. The obvious starting point for such a thing in Go was the [mafsa](https://github.com/smartystreets/mafsa) project. While working with mafsa I encountered some issues. First, it did not stream data to disk while building. Second, it chose to use a rune as the fundamental unit of transition in the FST, but I felt using a byte would be more powerful in the end. My hope is that higher-level encoding-aware traversals will be possible when necessary. Finally, as I reported bugs and submitted PRs I learned that the mafsa project was mainly a research project and no longer being maintained. I wanted to build something that could be used in production. As the project advanced more and more techniques from the [BurntSushi/fst](https://github.com/BurntSushi/fst) were adapted to our implementation. + +## Related Work + +Much credit goes to two existing projects: + - [mafsa](https://github.com/smartystreets/mafsa) + - [BurntSushi/fst](https://github.com/BurntSushi/fst) + +Most of the original implementation here started with my digging into the internals of mafsa. As the implementation progressed, I continued to borrow ideas/approaches from the BurntSushi/fst library as well. + +For a great introduction to this topic, please read the blog post [Index 1,600,000,000 Keys with Automata and Rust](http://blog.burntsushi.net/transducers/) diff --git a/vendor/github.com/couchbase/vellum/automaton.go b/vendor/github.com/couchbase/vellum/automaton.go new file mode 100644 index 00000000..47526595 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/automaton.go @@ -0,0 +1,85 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +// Automaton represents the general contract of a byte-based finite automaton +type Automaton interface { + + // Start returns the start state + Start() int + + // IsMatch returns true if and only if the state is a match + IsMatch(int) bool + + // CanMatch returns true if and only if it is possible to reach a match + // in zero or more steps + CanMatch(int) bool + + // WillAlwaysMatch returns true if and only if the current state matches + // and will always match no matter what steps are taken + WillAlwaysMatch(int) bool + + // Accept returns the next state given the input to the specified state + Accept(int, byte) int +} + +// AutomatonContains implements an generic Contains() method which works +// on any implementation of Automaton +func AutomatonContains(a Automaton, k []byte) bool { + i := 0 + curr := a.Start() + for a.CanMatch(curr) && i < len(k) { + curr = a.Accept(curr, k[i]) + if curr == noneAddr { + break + } + i++ + } + if i != len(k) { + return false + } + return a.IsMatch(curr) +} + +// AlwaysMatch is an Automaton implementation which always matches +type AlwaysMatch struct{} + +// Start returns the AlwaysMatch start state +func (m *AlwaysMatch) Start() int { + return 0 +} + +// IsMatch always returns true +func (m *AlwaysMatch) IsMatch(int) bool { + return true +} + +// CanMatch always returns true +func (m *AlwaysMatch) CanMatch(int) bool { + return true +} + +// WillAlwaysMatch always returns true +func (m *AlwaysMatch) WillAlwaysMatch(int) bool { + return true +} + +// Accept returns the next AlwaysMatch state +func (m *AlwaysMatch) Accept(int, byte) int { + return 0 +} + +// creating an alwaysMatchAutomaton to avoid unnecesary repeated allocations. +var alwaysMatchAutomaton = &AlwaysMatch{} diff --git a/vendor/github.com/couchbase/vellum/builder.go b/vendor/github.com/couchbase/vellum/builder.go new file mode 100644 index 00000000..b21db980 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/builder.go @@ -0,0 +1,453 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "bytes" + "io" +) + +var defaultBuilderOpts = &BuilderOpts{ + Encoder: 1, + RegistryTableSize: 10000, + RegistryMRUSize: 2, +} + +// A Builder is used to build a new FST. When possible data is +// streamed out to the underlying Writer as soon as possible. +type Builder struct { + unfinished *unfinishedNodes + registry *registry + last []byte + len int + + lastAddr int + + encoder encoder + opts *BuilderOpts + + builderNodePool builderNodePool + transitionPool transitionPool +} + +const noneAddr = 1 +const emptyAddr = 0 + +// NewBuilder returns a new Builder which will stream out the +// underlying representation to the provided Writer as the set is built. +func newBuilder(w io.Writer, opts *BuilderOpts) (*Builder, error) { + if opts == nil { + opts = defaultBuilderOpts + } + rv := &Builder{ + registry: newRegistry(opts.RegistryTableSize, opts.RegistryMRUSize), + opts: opts, + lastAddr: noneAddr, + } + rv.unfinished = newUnfinishedNodes(&rv.builderNodePool) + + var err error + rv.encoder, err = loadEncoder(opts.Encoder, w) + if err != nil { + return nil, err + } + err = rv.encoder.start() + if err != nil { + return nil, err + } + return rv, nil +} + +func (b *Builder) Reset(w io.Writer) error { + b.transitionPool.reset() + b.builderNodePool.reset() + b.unfinished.Reset(&b.builderNodePool) + b.registry.Reset() + b.lastAddr = noneAddr + b.encoder.reset(w) + b.last = nil + b.len = 0 + + err := b.encoder.start() + if err != nil { + return err + } + return nil +} + +// Insert the provided value to the set being built. +// NOTE: values must be inserted in lexicographical order. +func (b *Builder) Insert(key []byte, val uint64) error { + // ensure items are added in lexicographic order + if bytes.Compare(key, b.last) < 0 { + return ErrOutOfOrder + } + if len(key) == 0 { + b.len = 1 + b.unfinished.setRootOutput(val) + return nil + } + + prefixLen, out := b.unfinished.findCommonPrefixAndSetOutput(key, val) + b.len++ + err := b.compileFrom(prefixLen) + if err != nil { + return err + } + b.copyLastKey(key) + b.unfinished.addSuffix(key[prefixLen:], out, &b.builderNodePool) + + return nil +} + +func (b *Builder) copyLastKey(key []byte) { + if b.last == nil { + b.last = make([]byte, 0, 64) + } else { + b.last = b.last[:0] + } + b.last = append(b.last, key...) +} + +// Close MUST be called after inserting all values. +func (b *Builder) Close() error { + err := b.compileFrom(0) + if err != nil { + return err + } + root := b.unfinished.popRoot() + rootAddr, err := b.compile(root) + if err != nil { + return err + } + return b.encoder.finish(b.len, rootAddr) +} + +func (b *Builder) compileFrom(iState int) error { + addr := noneAddr + for iState+1 < len(b.unfinished.stack) { + var node *builderNode + if addr == noneAddr { + node = b.unfinished.popEmpty() + } else { + node = b.unfinished.popFreeze(addr, &b.transitionPool) + } + var err error + addr, err = b.compile(node) + if err != nil { + return nil + } + } + b.unfinished.topLastFreeze(addr, &b.transitionPool) + return nil +} + +func (b *Builder) compile(node *builderNode) (int, error) { + if node.final && len(node.trans) == 0 && + node.finalOutput == 0 { + return 0, nil + } + found, addr, entry := b.registry.entry(node) + if found { + return addr, nil + } + addr, err := b.encoder.encodeState(node, b.lastAddr) + if err != nil { + return 0, err + } + + b.lastAddr = addr + entry.addr = addr + return addr, nil +} + +type unfinishedNodes struct { + stack []*builderNodeUnfinished + + // cache allocates a reasonable number of builderNodeUnfinished + // objects up front and tries to keep reusing them + // because the main data structure is a stack, we assume the + // same access pattern, and don't track items separately + // this means calls get() and pushXYZ() must be paired, + // as well as calls put() and popXYZ() + cache []builderNodeUnfinished +} + +func (u *unfinishedNodes) Reset(p *builderNodePool) { + u.stack = u.stack[:0] + for i := 0; i < len(u.cache); i++ { + u.cache[i] = builderNodeUnfinished{} + } + u.pushEmpty(false, p) +} + +func newUnfinishedNodes(p *builderNodePool) *unfinishedNodes { + rv := &unfinishedNodes{ + stack: make([]*builderNodeUnfinished, 0, 64), + cache: make([]builderNodeUnfinished, 64), + } + rv.pushEmpty(false, p) + return rv +} + +// get new builderNodeUnfinished, reusing cache if possible +func (u *unfinishedNodes) get() *builderNodeUnfinished { + if len(u.stack) < len(u.cache) { + return &u.cache[len(u.stack)] + } + // full now allocate a new one + return &builderNodeUnfinished{} +} + +// return builderNodeUnfinished, clearing it for reuse +func (u *unfinishedNodes) put() { + if len(u.stack) >= len(u.cache) { + return + // do nothing, not part of cache + } + u.cache[len(u.stack)] = builderNodeUnfinished{} +} + +func (u *unfinishedNodes) findCommonPrefixAndSetOutput(key []byte, + out uint64) (int, uint64) { + var i int + for i < len(key) { + if i >= len(u.stack) { + break + } + var addPrefix uint64 + if !u.stack[i].hasLastT { + break + } + if u.stack[i].lastIn == key[i] { + commonPre := outputPrefix(u.stack[i].lastOut, out) + addPrefix = outputSub(u.stack[i].lastOut, commonPre) + out = outputSub(out, commonPre) + u.stack[i].lastOut = commonPre + i++ + } else { + break + } + + if addPrefix != 0 { + u.stack[i].addOutputPrefix(addPrefix) + } + } + + return i, out +} + +func (u *unfinishedNodes) pushEmpty(final bool, p *builderNodePool) { + next := u.get() + next.node = p.alloc() + next.node.final = final + u.stack = append(u.stack, next) +} + +func (u *unfinishedNodes) popRoot() *builderNode { + l := len(u.stack) + var unfinished *builderNodeUnfinished + u.stack, unfinished = u.stack[:l-1], u.stack[l-1] + rv := unfinished.node + u.put() + return rv +} + +func (u *unfinishedNodes) popFreeze(addr int, tp *transitionPool) *builderNode { + l := len(u.stack) + var unfinished *builderNodeUnfinished + u.stack, unfinished = u.stack[:l-1], u.stack[l-1] + unfinished.lastCompiled(addr, tp) + rv := unfinished.node + u.put() + return rv +} + +func (u *unfinishedNodes) popEmpty() *builderNode { + l := len(u.stack) + var unfinished *builderNodeUnfinished + u.stack, unfinished = u.stack[:l-1], u.stack[l-1] + rv := unfinished.node + u.put() + return rv +} + +func (u *unfinishedNodes) setRootOutput(out uint64) { + u.stack[0].node.final = true + u.stack[0].node.finalOutput = out +} + +func (u *unfinishedNodes) topLastFreeze(addr int, tp *transitionPool) { + last := len(u.stack) - 1 + u.stack[last].lastCompiled(addr, tp) +} + +func (u *unfinishedNodes) addSuffix(bs []byte, out uint64, p *builderNodePool) { + if len(bs) == 0 { + return + } + last := len(u.stack) - 1 + u.stack[last].hasLastT = true + u.stack[last].lastIn = bs[0] + u.stack[last].lastOut = out + for _, b := range bs[1:] { + next := u.get() + next.node = p.alloc() + next.hasLastT = true + next.lastIn = b + next.lastOut = 0 + u.stack = append(u.stack, next) + } + u.pushEmpty(true, p) +} + +type builderNodeUnfinished struct { + node *builderNode + lastOut uint64 + lastIn byte + hasLastT bool +} + +func (b *builderNodeUnfinished) lastCompiled(addr int, tp *transitionPool) { + if b.hasLastT { + transIn := b.lastIn + transOut := b.lastOut + b.hasLastT = false + b.lastOut = 0 + trans := tp.alloc() + trans.in = transIn + trans.out = transOut + trans.addr = addr + b.node.trans = append(b.node.trans, trans) + } +} + +func (b *builderNodeUnfinished) addOutputPrefix(prefix uint64) { + if b.node.final { + b.node.finalOutput = outputCat(prefix, b.node.finalOutput) + } + for _, t := range b.node.trans { + t.out = outputCat(prefix, t.out) + } + if b.hasLastT { + b.lastOut = outputCat(prefix, b.lastOut) + } +} + +type builderNode struct { + finalOutput uint64 + trans []*transition + final bool +} + +func (n *builderNode) equiv(o *builderNode) bool { + if n.final != o.final { + return false + } + if n.finalOutput != o.finalOutput { + return false + } + if len(n.trans) != len(o.trans) { + return false + } + for i, ntrans := range n.trans { + otrans := o.trans[i] + if ntrans.in != otrans.in { + return false + } + if ntrans.addr != otrans.addr { + return false + } + if ntrans.out != otrans.out { + return false + } + } + return true +} + +type transition struct { + out uint64 + addr int + in byte +} + +func outputPrefix(l, r uint64) uint64 { + if l < r { + return l + } + return r +} + +func outputSub(l, r uint64) uint64 { + return l - r +} + +func outputCat(l, r uint64) uint64 { + return l + r +} + +// the next builderNode to alloc() will be all[nextOuter][nextInner] +type builderNodePool struct { + all [][]builderNode + nextOuter int + nextInner int +} + +func (p *builderNodePool) reset() { + p.nextOuter = 0 + p.nextInner = 0 +} + +func (p *builderNodePool) alloc() *builderNode { + if p.nextOuter >= len(p.all) { + p.all = append(p.all, make([]builderNode, 256)) + } + rv := &p.all[p.nextOuter][p.nextInner] + p.nextInner += 1 + if p.nextInner >= len(p.all[p.nextOuter]) { + p.nextOuter += 1 + p.nextInner = 0 + } + rv.finalOutput = 0 + rv.trans = rv.trans[:0] + rv.final = false + return rv +} + +// the next transition to alloc() will be all[nextOuter][nextInner] +type transitionPool struct { + all [][]transition + nextOuter int + nextInner int +} + +func (p *transitionPool) reset() { + p.nextOuter = 0 + p.nextInner = 0 +} + +func (p *transitionPool) alloc() *transition { + if p.nextOuter >= len(p.all) { + p.all = append(p.all, make([]transition, 256)) + } + rv := &p.all[p.nextOuter][p.nextInner] + p.nextInner += 1 + if p.nextInner >= len(p.all[p.nextOuter]) { + p.nextOuter += 1 + p.nextInner = 0 + } + *rv = transition{} + return rv +} diff --git a/vendor/github.com/couchbase/vellum/common.go b/vendor/github.com/couchbase/vellum/common.go new file mode 100644 index 00000000..cd3e6a0d --- /dev/null +++ b/vendor/github.com/couchbase/vellum/common.go @@ -0,0 +1,547 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +const maxCommon = 1<<6 - 1 + +func encodeCommon(in byte) byte { + val := byte((int(commonInputs[in]) + 1) % 256) + if val > maxCommon { + return 0 + } + return val +} + +func decodeCommon(in byte) byte { + return commonInputsInv[in-1] +} + +var commonInputs = []byte{ + 84, // '\x00' + 85, // '\x01' + 86, // '\x02' + 87, // '\x03' + 88, // '\x04' + 89, // '\x05' + 90, // '\x06' + 91, // '\x07' + 92, // '\x08' + 93, // '\t' + 94, // '\n' + 95, // '\x0b' + 96, // '\x0c' + 97, // '\r' + 98, // '\x0e' + 99, // '\x0f' + 100, // '\x10' + 101, // '\x11' + 102, // '\x12' + 103, // '\x13' + 104, // '\x14' + 105, // '\x15' + 106, // '\x16' + 107, // '\x17' + 108, // '\x18' + 109, // '\x19' + 110, // '\x1a' + 111, // '\x1b' + 112, // '\x1c' + 113, // '\x1d' + 114, // '\x1e' + 115, // '\x1f' + 116, // ' ' + 80, // '!' + 117, // '"' + 118, // '#' + 79, // '$' + 39, // '%' + 30, // '&' + 81, // "'" + 75, // '(' + 74, // ')' + 82, // '*' + 57, // '+' + 66, // ',' + 16, // '-' + 12, // '.' + 2, // '/' + 19, // '0' + 20, // '1' + 21, // '2' + 27, // '3' + 32, // '4' + 29, // '5' + 35, // '6' + 36, // '7' + 37, // '8' + 34, // '9' + 24, // ':' + 73, // ';' + 119, // '<' + 23, // '=' + 120, // '>' + 40, // '?' + 83, // '@' + 44, // 'A' + 48, // 'B' + 42, // 'C' + 43, // 'D' + 49, // 'E' + 46, // 'F' + 62, // 'G' + 61, // 'H' + 47, // 'I' + 69, // 'J' + 68, // 'K' + 58, // 'L' + 56, // 'M' + 55, // 'N' + 59, // 'O' + 51, // 'P' + 72, // 'Q' + 54, // 'R' + 45, // 'S' + 52, // 'T' + 64, // 'U' + 65, // 'V' + 63, // 'W' + 71, // 'X' + 67, // 'Y' + 70, // 'Z' + 77, // '[' + 121, // '\\' + 78, // ']' + 122, // '^' + 31, // '_' + 123, // '`' + 4, // 'a' + 25, // 'b' + 9, // 'c' + 17, // 'd' + 1, // 'e' + 26, // 'f' + 22, // 'g' + 13, // 'h' + 7, // 'i' + 50, // 'j' + 38, // 'k' + 14, // 'l' + 15, // 'm' + 10, // 'n' + 3, // 'o' + 8, // 'p' + 60, // 'q' + 6, // 'r' + 5, // 's' + 0, // 't' + 18, // 'u' + 33, // 'v' + 11, // 'w' + 41, // 'x' + 28, // 'y' + 53, // 'z' + 124, // '{' + 125, // '|' + 126, // '}' + 76, // '~' + 127, // '\x7f' + 128, // '\x80' + 129, // '\x81' + 130, // '\x82' + 131, // '\x83' + 132, // '\x84' + 133, // '\x85' + 134, // '\x86' + 135, // '\x87' + 136, // '\x88' + 137, // '\x89' + 138, // '\x8a' + 139, // '\x8b' + 140, // '\x8c' + 141, // '\x8d' + 142, // '\x8e' + 143, // '\x8f' + 144, // '\x90' + 145, // '\x91' + 146, // '\x92' + 147, // '\x93' + 148, // '\x94' + 149, // '\x95' + 150, // '\x96' + 151, // '\x97' + 152, // '\x98' + 153, // '\x99' + 154, // '\x9a' + 155, // '\x9b' + 156, // '\x9c' + 157, // '\x9d' + 158, // '\x9e' + 159, // '\x9f' + 160, // '\xa0' + 161, // '¡' + 162, // '¢' + 163, // '£' + 164, // '¤' + 165, // '¥' + 166, // '¦' + 167, // '§' + 168, // '¨' + 169, // '©' + 170, // 'ª' + 171, // '«' + 172, // '¬' + 173, // '\xad' + 174, // '®' + 175, // '¯' + 176, // '°' + 177, // '±' + 178, // '²' + 179, // '³' + 180, // '´' + 181, // 'µ' + 182, // '¶' + 183, // '·' + 184, // '¸' + 185, // '¹' + 186, // 'º' + 187, // '»' + 188, // '¼' + 189, // '½' + 190, // '¾' + 191, // '¿' + 192, // 'À' + 193, // 'Á' + 194, // 'Â' + 195, // 'Ã' + 196, // 'Ä' + 197, // 'Å' + 198, // 'Æ' + 199, // 'Ç' + 200, // 'È' + 201, // 'É' + 202, // 'Ê' + 203, // 'Ë' + 204, // 'Ì' + 205, // 'Í' + 206, // 'Î' + 207, // 'Ï' + 208, // 'Ð' + 209, // 'Ñ' + 210, // 'Ò' + 211, // 'Ó' + 212, // 'Ô' + 213, // 'Õ' + 214, // 'Ö' + 215, // '×' + 216, // 'Ø' + 217, // 'Ù' + 218, // 'Ú' + 219, // 'Û' + 220, // 'Ü' + 221, // 'Ý' + 222, // 'Þ' + 223, // 'ß' + 224, // 'à' + 225, // 'á' + 226, // 'â' + 227, // 'ã' + 228, // 'ä' + 229, // 'å' + 230, // 'æ' + 231, // 'ç' + 232, // 'è' + 233, // 'é' + 234, // 'ê' + 235, // 'ë' + 236, // 'ì' + 237, // 'í' + 238, // 'î' + 239, // 'ï' + 240, // 'ð' + 241, // 'ñ' + 242, // 'ò' + 243, // 'ó' + 244, // 'ô' + 245, // 'õ' + 246, // 'ö' + 247, // '÷' + 248, // 'ø' + 249, // 'ù' + 250, // 'ú' + 251, // 'û' + 252, // 'ü' + 253, // 'ý' + 254, // 'þ' + 255, // 'ÿ' +} + +var commonInputsInv = []byte{ + 't', + 'e', + '/', + 'o', + 'a', + 's', + 'r', + 'i', + 'p', + 'c', + 'n', + 'w', + '.', + 'h', + 'l', + 'm', + '-', + 'd', + 'u', + '0', + '1', + '2', + 'g', + '=', + ':', + 'b', + 'f', + '3', + 'y', + '5', + '&', + '_', + '4', + 'v', + '9', + '6', + '7', + '8', + 'k', + '%', + '?', + 'x', + 'C', + 'D', + 'A', + 'S', + 'F', + 'I', + 'B', + 'E', + 'j', + 'P', + 'T', + 'z', + 'R', + 'N', + 'M', + '+', + 'L', + 'O', + 'q', + 'H', + 'G', + 'W', + 'U', + 'V', + ',', + 'Y', + 'K', + 'J', + 'Z', + 'X', + 'Q', + ';', + ')', + '(', + '~', + '[', + ']', + '$', + '!', + '\'', + '*', + '@', + '\x00', + '\x01', + '\x02', + '\x03', + '\x04', + '\x05', + '\x06', + '\x07', + '\x08', + '\t', + '\n', + '\x0b', + '\x0c', + '\r', + '\x0e', + '\x0f', + '\x10', + '\x11', + '\x12', + '\x13', + '\x14', + '\x15', + '\x16', + '\x17', + '\x18', + '\x19', + '\x1a', + '\x1b', + '\x1c', + '\x1d', + '\x1e', + '\x1f', + ' ', + '"', + '#', + '<', + '>', + '\\', + '^', + '`', + '{', + '|', + '}', + '\x7f', + '\x80', + '\x81', + '\x82', + '\x83', + '\x84', + '\x85', + '\x86', + '\x87', + '\x88', + '\x89', + '\x8a', + '\x8b', + '\x8c', + '\x8d', + '\x8e', + '\x8f', + '\x90', + '\x91', + '\x92', + '\x93', + '\x94', + '\x95', + '\x96', + '\x97', + '\x98', + '\x99', + '\x9a', + '\x9b', + '\x9c', + '\x9d', + '\x9e', + '\x9f', + '\xa0', + '\xa1', + '\xa2', + '\xa3', + '\xa4', + '\xa5', + '\xa6', + '\xa7', + '\xa8', + '\xa9', + '\xaa', + '\xab', + '\xac', + '\xad', + '\xae', + '\xaf', + '\xb0', + '\xb1', + '\xb2', + '\xb3', + '\xb4', + '\xb5', + '\xb6', + '\xb7', + '\xb8', + '\xb9', + '\xba', + '\xbb', + '\xbc', + '\xbd', + '\xbe', + '\xbf', + '\xc0', + '\xc1', + '\xc2', + '\xc3', + '\xc4', + '\xc5', + '\xc6', + '\xc7', + '\xc8', + '\xc9', + '\xca', + '\xcb', + '\xcc', + '\xcd', + '\xce', + '\xcf', + '\xd0', + '\xd1', + '\xd2', + '\xd3', + '\xd4', + '\xd5', + '\xd6', + '\xd7', + '\xd8', + '\xd9', + '\xda', + '\xdb', + '\xdc', + '\xdd', + '\xde', + '\xdf', + '\xe0', + '\xe1', + '\xe2', + '\xe3', + '\xe4', + '\xe5', + '\xe6', + '\xe7', + '\xe8', + '\xe9', + '\xea', + '\xeb', + '\xec', + '\xed', + '\xee', + '\xef', + '\xf0', + '\xf1', + '\xf2', + '\xf3', + '\xf4', + '\xf5', + '\xf6', + '\xf7', + '\xf8', + '\xf9', + '\xfa', + '\xfb', + '\xfc', + '\xfd', + '\xfe', + '\xff', +} diff --git a/vendor/github.com/couchbase/vellum/decoder_v1.go b/vendor/github.com/couchbase/vellum/decoder_v1.go new file mode 100644 index 00000000..5a0ea688 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/decoder_v1.go @@ -0,0 +1,316 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "bytes" + "encoding/binary" + "fmt" + "strconv" +) + +func init() { + registerDecoder(versionV1, func(data []byte) decoder { + return newDecoderV1(data) + }) +} + +type decoderV1 struct { + data []byte + root uint64 + len uint64 +} + +func newDecoderV1(data []byte) *decoderV1 { + return &decoderV1{ + data: data, + } +} + +func (d *decoderV1) getRoot() int { + if len(d.data) < footerSizeV1 { + return noneAddr + } + footer := d.data[len(d.data)-footerSizeV1:] + root := binary.LittleEndian.Uint64(footer[8:]) + return int(root) +} + +func (d *decoderV1) getLen() int { + if len(d.data) < footerSizeV1 { + return 0 + } + footer := d.data[len(d.data)-footerSizeV1:] + dlen := binary.LittleEndian.Uint64(footer) + return int(dlen) +} + +func (d *decoderV1) stateAt(addr int, prealloc fstState) (fstState, error) { + state, ok := prealloc.(*fstStateV1) + if ok && state != nil { + *state = fstStateV1{} // clear the struct + } else { + state = &fstStateV1{} + } + err := state.at(d.data, addr) + if err != nil { + return nil, err + } + return state, nil +} + +type fstStateV1 struct { + data []byte + top int + bottom int + numTrans int + + // single trans only + singleTransChar byte + singleTransNext bool + singleTransAddr uint64 + singleTransOut uint64 + + // shared + transSize int + outSize int + + // multiple trans only + final bool + transTop int + transBottom int + destTop int + destBottom int + outTop int + outBottom int + outFinal int +} + +func (f *fstStateV1) isEncodedSingle() bool { + if f.data[f.top]>>7 > 0 { + return true + } + return false +} + +func (f *fstStateV1) at(data []byte, addr int) error { + f.data = data + if addr == emptyAddr { + return f.atZero() + } else if addr == noneAddr { + return f.atNone() + } + if addr > len(data) || addr < 16 { + return fmt.Errorf("invalid address %d/%d", addr, len(data)) + } + f.top = addr + f.bottom = addr + if f.isEncodedSingle() { + return f.atSingle(data, addr) + } + return f.atMulti(data, addr) +} + +func (f *fstStateV1) atZero() error { + f.top = 0 + f.bottom = 1 + f.numTrans = 0 + f.final = true + f.outFinal = 0 + return nil +} + +func (f *fstStateV1) atNone() error { + f.top = 0 + f.bottom = 1 + f.numTrans = 0 + f.final = false + f.outFinal = 0 + return nil +} + +func (f *fstStateV1) atSingle(data []byte, addr int) error { + // handle single transition case + f.numTrans = 1 + f.singleTransNext = data[f.top]&transitionNext > 0 + f.singleTransChar = data[f.top] & maxCommon + if f.singleTransChar == 0 { + f.bottom-- // extra byte for uncommon + f.singleTransChar = data[f.bottom] + } else { + f.singleTransChar = decodeCommon(f.singleTransChar) + } + if f.singleTransNext { + // now we know the bottom, can compute next addr + f.singleTransAddr = uint64(f.bottom - 1) + f.singleTransOut = 0 + } else { + f.bottom-- // extra byte with pack sizes + f.transSize, f.outSize = decodePackSize(data[f.bottom]) + f.bottom -= f.transSize // exactly one trans + f.singleTransAddr = readPackedUint(data[f.bottom : f.bottom+f.transSize]) + if f.outSize > 0 { + f.bottom -= f.outSize // exactly one out (could be length 0 though) + f.singleTransOut = readPackedUint(data[f.bottom : f.bottom+f.outSize]) + } else { + f.singleTransOut = 0 + } + // need to wait till we know bottom + if f.singleTransAddr != 0 { + f.singleTransAddr = uint64(f.bottom) - f.singleTransAddr + } + } + return nil +} + +func (f *fstStateV1) atMulti(data []byte, addr int) error { + // handle multiple transitions case + f.final = data[f.top]&stateFinal > 0 + f.numTrans = int(data[f.top] & maxNumTrans) + if f.numTrans == 0 { + f.bottom-- // extra byte for number of trans + f.numTrans = int(data[f.bottom]) + if f.numTrans == 1 { + // can't really be 1 here, this is special case that means 256 + f.numTrans = 256 + } + } + f.bottom-- // extra byte with pack sizes + f.transSize, f.outSize = decodePackSize(data[f.bottom]) + + f.transTop = f.bottom + f.bottom -= f.numTrans // one byte for each transition + f.transBottom = f.bottom + + f.destTop = f.bottom + f.bottom -= f.numTrans * f.transSize + f.destBottom = f.bottom + + if f.outSize > 0 { + f.outTop = f.bottom + f.bottom -= f.numTrans * f.outSize + f.outBottom = f.bottom + if f.final { + f.bottom -= f.outSize + f.outFinal = f.bottom + } + } + return nil +} + +func (f *fstStateV1) Address() int { + return f.top +} + +func (f *fstStateV1) Final() bool { + return f.final +} + +func (f *fstStateV1) FinalOutput() uint64 { + if f.numTrans > 0 && f.final && f.outSize > 0 { + return readPackedUint(f.data[f.outFinal : f.outFinal+f.outSize]) + } + return 0 +} + +func (f *fstStateV1) NumTransitions() int { + return f.numTrans +} + +func (f *fstStateV1) TransitionAt(i int) byte { + if f.isEncodedSingle() { + return f.singleTransChar + } + transitionKeys := f.data[f.transBottom:f.transTop] + return transitionKeys[f.numTrans-i-1] +} + +func (f *fstStateV1) TransitionFor(b byte) (int, int, uint64) { + if f.isEncodedSingle() { + if f.singleTransChar == b { + return 0, int(f.singleTransAddr), f.singleTransOut + } + return -1, noneAddr, 0 + } + transitionKeys := f.data[f.transBottom:f.transTop] + pos := bytes.IndexByte(transitionKeys, b) + if pos < 0 { + return -1, noneAddr, 0 + } + transDests := f.data[f.destBottom:f.destTop] + dest := int(readPackedUint(transDests[pos*f.transSize : pos*f.transSize+f.transSize])) + if dest > 0 { + // convert delta + dest = f.bottom - dest + } + transVals := f.data[f.outBottom:f.outTop] + var out uint64 + if f.outSize > 0 { + out = readPackedUint(transVals[pos*f.outSize : pos*f.outSize+f.outSize]) + } + return f.numTrans - pos - 1, dest, out +} + +func (f *fstStateV1) String() string { + rv := "" + rv += fmt.Sprintf("State: %d (%#x)", f.top, f.top) + if f.final { + rv += " final" + fout := f.FinalOutput() + if fout != 0 { + rv += fmt.Sprintf(" (%d)", fout) + } + } + rv += "\n" + rv += fmt.Sprintf("Data: % x\n", f.data[f.bottom:f.top+1]) + + for i := 0; i < f.numTrans; i++ { + transChar := f.TransitionAt(i) + _, transDest, transOut := f.TransitionFor(transChar) + rv += fmt.Sprintf(" - %d (%#x) '%s' ---> %d (%#x) with output: %d", transChar, transChar, string(transChar), transDest, transDest, transOut) + rv += "\n" + } + if f.numTrans == 0 { + rv += "\n" + } + return rv +} + +func (f *fstStateV1) DotString(num int) string { + rv := "" + label := fmt.Sprintf("%d", num) + final := "" + if f.final { + final = ",peripheries=2" + } + rv += fmt.Sprintf(" %d [label=\"%s\"%s];\n", f.top, label, final) + + for i := 0; i < f.numTrans; i++ { + transChar := f.TransitionAt(i) + _, transDest, transOut := f.TransitionFor(transChar) + out := "" + if transOut != 0 { + out = fmt.Sprintf("/%d", transOut) + } + rv += fmt.Sprintf(" %d -> %d [label=\"%s%s\"];\n", f.top, transDest, escapeInput(transChar), out) + } + + return rv +} + +func escapeInput(b byte) string { + x := strconv.AppendQuoteRune(nil, rune(b)) + return string(x[1:(len(x) - 1)]) +} diff --git a/vendor/github.com/couchbase/vellum/encoder_v1.go b/vendor/github.com/couchbase/vellum/encoder_v1.go new file mode 100644 index 00000000..0651fc86 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/encoder_v1.go @@ -0,0 +1,227 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "encoding/binary" + "fmt" + "io" +) + +const versionV1 = 1 +const oneTransition = 1 << 7 +const transitionNext = 1 << 6 +const stateFinal = 1 << 6 +const footerSizeV1 = 16 + +func init() { + registerEncoder(versionV1, func(w io.Writer) encoder { + return newEncoderV1(w) + }) +} + +type encoderV1 struct { + bw *writer +} + +func newEncoderV1(w io.Writer) *encoderV1 { + return &encoderV1{ + bw: newWriter(w), + } +} + +func (e *encoderV1) reset(w io.Writer) { + e.bw.Reset(w) +} + +func (e *encoderV1) start() error { + header := make([]byte, headerSize) + binary.LittleEndian.PutUint64(header, versionV1) + binary.LittleEndian.PutUint64(header[8:], uint64(0)) // type + n, err := e.bw.Write(header) + if err != nil { + return err + } + if n != headerSize { + return fmt.Errorf("short write of header %d/%d", n, headerSize) + } + return nil +} + +func (e *encoderV1) encodeState(s *builderNode, lastAddr int) (int, error) { + if len(s.trans) == 0 && s.final && s.finalOutput == 0 { + return 0, nil + } else if len(s.trans) != 1 || s.final { + return e.encodeStateMany(s) + } else if !s.final && s.trans[0].out == 0 && s.trans[0].addr == lastAddr { + return e.encodeStateOneFinish(s, transitionNext) + } + return e.encodeStateOne(s) +} + +func (e *encoderV1) encodeStateOne(s *builderNode) (int, error) { + start := uint64(e.bw.counter) + outPackSize := 0 + if s.trans[0].out != 0 { + outPackSize = packedSize(s.trans[0].out) + err := e.bw.WritePackedUintIn(s.trans[0].out, outPackSize) + if err != nil { + return 0, err + } + } + delta := deltaAddr(start, uint64(s.trans[0].addr)) + transPackSize := packedSize(delta) + err := e.bw.WritePackedUintIn(delta, transPackSize) + if err != nil { + return 0, err + } + + packSize := encodePackSize(transPackSize, outPackSize) + err = e.bw.WriteByte(packSize) + if err != nil { + return 0, err + } + + return e.encodeStateOneFinish(s, 0) +} + +func (e *encoderV1) encodeStateOneFinish(s *builderNode, next byte) (int, error) { + enc := encodeCommon(s.trans[0].in) + + // not a common input + if enc == 0 { + err := e.bw.WriteByte(s.trans[0].in) + if err != nil { + return 0, err + } + } + err := e.bw.WriteByte(oneTransition | next | enc) + if err != nil { + return 0, err + } + + return e.bw.counter - 1, nil +} + +func (e *encoderV1) encodeStateMany(s *builderNode) (int, error) { + start := uint64(e.bw.counter) + transPackSize := 0 + outPackSize := packedSize(s.finalOutput) + anyOutputs := s.finalOutput != 0 + for i := range s.trans { + delta := deltaAddr(start, uint64(s.trans[i].addr)) + tsize := packedSize(delta) + if tsize > transPackSize { + transPackSize = tsize + } + osize := packedSize(s.trans[i].out) + if osize > outPackSize { + outPackSize = osize + } + anyOutputs = anyOutputs || s.trans[i].out != 0 + } + if !anyOutputs { + outPackSize = 0 + } + + if anyOutputs { + // output final value + if s.final { + err := e.bw.WritePackedUintIn(s.finalOutput, outPackSize) + if err != nil { + return 0, err + } + } + // output transition values (in reverse) + for j := len(s.trans) - 1; j >= 0; j-- { + err := e.bw.WritePackedUintIn(s.trans[j].out, outPackSize) + if err != nil { + return 0, err + } + } + } + + // output transition dests (in reverse) + for j := len(s.trans) - 1; j >= 0; j-- { + delta := deltaAddr(start, uint64(s.trans[j].addr)) + err := e.bw.WritePackedUintIn(delta, transPackSize) + if err != nil { + return 0, err + } + } + + // output transition keys (in reverse) + for j := len(s.trans) - 1; j >= 0; j-- { + err := e.bw.WriteByte(s.trans[j].in) + if err != nil { + return 0, err + } + } + + packSize := encodePackSize(transPackSize, outPackSize) + err := e.bw.WriteByte(packSize) + if err != nil { + return 0, err + } + + numTrans := encodeNumTrans(len(s.trans)) + + // if number of transitions wont fit in edge header byte + // write out separately + if numTrans == 0 { + if len(s.trans) == 256 { + // this wouldn't fit in single byte, but reuse value 1 + // which would have always fit in the edge header instead + err = e.bw.WriteByte(1) + if err != nil { + return 0, err + } + } else { + err = e.bw.WriteByte(byte(len(s.trans))) + if err != nil { + return 0, err + } + } + } + + // finally write edge header + if s.final { + numTrans |= stateFinal + } + err = e.bw.WriteByte(numTrans) + if err != nil { + return 0, err + } + + return e.bw.counter - 1, nil +} + +func (e *encoderV1) finish(count, rootAddr int) error { + footer := make([]byte, footerSizeV1) + binary.LittleEndian.PutUint64(footer, uint64(count)) // root addr + binary.LittleEndian.PutUint64(footer[8:], uint64(rootAddr)) // root addr + n, err := e.bw.Write(footer) + if err != nil { + return err + } + if n != footerSizeV1 { + return fmt.Errorf("short write of footer %d/%d", n, footerSizeV1) + } + err = e.bw.Flush() + if err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/couchbase/vellum/encoding.go b/vendor/github.com/couchbase/vellum/encoding.go new file mode 100644 index 00000000..988d4864 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/encoding.go @@ -0,0 +1,87 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "encoding/binary" + "fmt" + "io" +) + +const headerSize = 16 + +type encoderConstructor func(w io.Writer) encoder +type decoderConstructor func([]byte) decoder + +var encoders = map[int]encoderConstructor{} +var decoders = map[int]decoderConstructor{} + +type encoder interface { + start() error + encodeState(s *builderNode, addr int) (int, error) + finish(count, rootAddr int) error + reset(w io.Writer) +} + +func loadEncoder(ver int, w io.Writer) (encoder, error) { + if cons, ok := encoders[ver]; ok { + return cons(w), nil + } + return nil, fmt.Errorf("no encoder for version %d registered", ver) +} + +func registerEncoder(ver int, cons encoderConstructor) { + encoders[ver] = cons +} + +type decoder interface { + getRoot() int + getLen() int + stateAt(addr int, prealloc fstState) (fstState, error) +} + +func loadDecoder(ver int, data []byte) (decoder, error) { + if cons, ok := decoders[ver]; ok { + return cons(data), nil + } + return nil, fmt.Errorf("no decoder for version %d registered", ver) +} + +func registerDecoder(ver int, cons decoderConstructor) { + decoders[ver] = cons +} + +func decodeHeader(header []byte) (ver int, typ int, err error) { + if len(header) < headerSize { + err = fmt.Errorf("invalid header < 16 bytes") + return + } + ver = int(binary.LittleEndian.Uint64(header[0:8])) + typ = int(binary.LittleEndian.Uint64(header[8:16])) + return +} + +// fstState represents a state inside the FTS runtime +// It is the main contract between the FST impl and the decoder +// The FST impl should work only with this interface, while only the decoder +// impl knows the physical representation. +type fstState interface { + Address() int + Final() bool + FinalOutput() uint64 + NumTransitions() int + TransitionFor(b byte) (int, int, uint64) + TransitionAt(i int) byte +} diff --git a/vendor/github.com/couchbase/vellum/fst.go b/vendor/github.com/couchbase/vellum/fst.go new file mode 100644 index 00000000..ecc52839 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/fst.go @@ -0,0 +1,254 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "io" + + "github.com/willf/bitset" +) + +// FST is an in-memory representation of a finite state transducer, +// capable of returning the uint64 value associated with +// each []byte key stored, as well as enumerating all of the keys +// in order. +type FST struct { + f io.Closer + ver int + len int + typ int + data []byte + decoder decoder +} + +func new(data []byte, f io.Closer) (rv *FST, err error) { + rv = &FST{ + data: data, + f: f, + } + + rv.ver, rv.typ, err = decodeHeader(data) + if err != nil { + return nil, err + } + + rv.decoder, err = loadDecoder(rv.ver, rv.data) + if err != nil { + return nil, err + } + + rv.len = rv.decoder.getLen() + + return rv, nil +} + +// Contains returns true if this FST contains the specified key. +func (f *FST) Contains(val []byte) (bool, error) { + _, exists, err := f.Get(val) + return exists, err +} + +// Get returns the value associated with the key. NOTE: a value of zero +// does not imply the key does not exist, you must consult the second +// return value as well. +func (f *FST) Get(input []byte) (uint64, bool, error) { + return f.get(input, nil) +} + +func (f *FST) get(input []byte, prealloc fstState) (uint64, bool, error) { + var total uint64 + curr := f.decoder.getRoot() + state, err := f.decoder.stateAt(curr, prealloc) + if err != nil { + return 0, false, err + } + for i := range input { + _, curr, output := state.TransitionFor(input[i]) + if curr == noneAddr { + return 0, false, nil + } + + state, err = f.decoder.stateAt(curr, state) + if err != nil { + return 0, false, err + } + + total += output + } + + if state.Final() { + total += state.FinalOutput() + return total, true, nil + } + return 0, false, nil +} + +// Version returns the encoding version used by this FST instance. +func (f *FST) Version() int { + return f.ver +} + +// Len returns the number of entries in this FST instance. +func (f *FST) Len() int { + return f.len +} + +// Type returns the type of this FST instance. +func (f *FST) Type() int { + return f.typ +} + +// Close will unmap any mmap'd data (if managed by vellum) and it will close +// the backing file (if managed by vellum). You MUST call Close() for any +// FST instance that is created. +func (f *FST) Close() error { + if f.f != nil { + err := f.f.Close() + if err != nil { + return err + } + } + f.data = nil + f.decoder = nil + return nil +} + +// Start returns the start state of this Automaton +func (f *FST) Start() int { + return f.decoder.getRoot() +} + +// IsMatch returns if this state is a matching state in this Automaton +func (f *FST) IsMatch(addr int) bool { + match, _ := f.IsMatchWithVal(addr) + return match +} + +// CanMatch returns if this state can ever transition to a matching state +// in this Automaton +func (f *FST) CanMatch(addr int) bool { + if addr == noneAddr { + return false + } + return true +} + +// WillAlwaysMatch returns if from this state the Automaton will always +// be in a matching state +func (f *FST) WillAlwaysMatch(int) bool { + return false +} + +// Accept returns the next state for this Automaton on input of byte b +func (f *FST) Accept(addr int, b byte) int { + next, _ := f.AcceptWithVal(addr, b) + return next +} + +// IsMatchWithVal returns if this state is a matching state in this Automaton +// and also returns the final output value for this state +func (f *FST) IsMatchWithVal(addr int) (bool, uint64) { + s, err := f.decoder.stateAt(addr, nil) + if err != nil { + return false, 0 + } + return s.Final(), s.FinalOutput() +} + +// AcceptWithVal returns the next state for this Automaton on input of byte b +// and also returns the output value for the transition +func (f *FST) AcceptWithVal(addr int, b byte) (int, uint64) { + s, err := f.decoder.stateAt(addr, nil) + if err != nil { + return noneAddr, 0 + } + _, next, output := s.TransitionFor(b) + return next, output +} + +// Iterator returns a new Iterator capable of enumerating the key/value pairs +// between the provided startKeyInclusive and endKeyExclusive. +func (f *FST) Iterator(startKeyInclusive, endKeyExclusive []byte) (*FSTIterator, error) { + return newIterator(f, startKeyInclusive, endKeyExclusive, nil) +} + +// Search returns a new Iterator capable of enumerating the key/value pairs +// between the provided startKeyInclusive and endKeyExclusive that also +// satisfy the provided automaton. +func (f *FST) Search(aut Automaton, startKeyInclusive, endKeyExclusive []byte) (*FSTIterator, error) { + return newIterator(f, startKeyInclusive, endKeyExclusive, aut) +} + +// Debug is only intended for debug purposes, it simply asks the underlying +// decoder visit each state, and pass it to the provided callback. +func (f *FST) Debug(callback func(int, interface{}) error) error { + + addr := f.decoder.getRoot() + set := bitset.New(uint(addr)) + stack := addrStack{addr} + + stateNumber := 0 + stack, addr = stack[:len(stack)-1], stack[len(stack)-1] + for addr != noneAddr { + if set.Test(uint(addr)) { + stack, addr = stack.Pop() + continue + } + set.Set(uint(addr)) + state, err := f.decoder.stateAt(addr, nil) + if err != nil { + return err + } + err = callback(stateNumber, state) + if err != nil { + return err + } + for i := 0; i < state.NumTransitions(); i++ { + tchar := state.TransitionAt(i) + _, dest, _ := state.TransitionFor(tchar) + stack = append(stack, dest) + } + stateNumber++ + stack, addr = stack.Pop() + } + + return nil +} + +type addrStack []int + +func (a addrStack) Pop() (addrStack, int) { + l := len(a) + if l < 1 { + return a, noneAddr + } + return a[:l-1], a[l-1] +} + +// Reader() returns a Reader instance that a single thread may use to +// retrieve data from the FST +func (f *FST) Reader() (*Reader, error) { + return &Reader{f: f}, nil +} + +// A Reader is meant for a single threaded use +type Reader struct { + f *FST + prealloc fstStateV1 +} + +func (r *Reader) Get(input []byte) (uint64, bool, error) { + return r.f.get(input, &r.prealloc) +} diff --git a/vendor/github.com/couchbase/vellum/fst_iterator.go b/vendor/github.com/couchbase/vellum/fst_iterator.go new file mode 100644 index 00000000..389ac64a --- /dev/null +++ b/vendor/github.com/couchbase/vellum/fst_iterator.go @@ -0,0 +1,276 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "bytes" +) + +// Iterator represents a means of visity key/value pairs in order. +type Iterator interface { + + // Current() returns the key/value pair currently pointed to. + // The []byte of the key is ONLY guaranteed to be valid until + // another call to Next/Seek/Close. If you need it beyond that + // point you MUST make a copy. + Current() ([]byte, uint64) + + // Next() advances the iterator to the next key/value pair. + // If no more key/value pairs exist, ErrIteratorDone is returned. + Next() error + + // Seek() advances the iterator the specified key, or the next key + // if it does not exist. + // If no keys exist after that point, ErrIteratorDone is returned. + Seek(key []byte) error + + // Reset resets the Iterator' internal state to allow for iterator + // reuse (e.g. pooling). + Reset(f *FST, startKeyInclusive, endKeyExclusive []byte, aut Automaton) error + + // Close() frees any resources held by this iterator. + Close() error +} + +// FSTIterator is a structure for iterating key/value pairs in this FST in +// lexicographic order. Iterators should be constructed with the FSTIterator +// method on the parent FST structure. +type FSTIterator struct { + f *FST + aut Automaton + + startKeyInclusive []byte + endKeyExclusive []byte + + statesStack []fstState + keysStack []byte + keysPosStack []int + valsStack []uint64 + autStatesStack []int + + nextStart []byte +} + +func newIterator(f *FST, startKeyInclusive, endKeyExclusive []byte, + aut Automaton) (*FSTIterator, error) { + + rv := &FSTIterator{} + err := rv.Reset(f, startKeyInclusive, endKeyExclusive, aut) + if err != nil { + return nil, err + } + return rv, nil +} + +// Reset resets the Iterator' internal state to allow for iterator +// reuse (e.g. pooling). +func (i *FSTIterator) Reset(f *FST, startKeyInclusive, endKeyExclusive []byte, aut Automaton) error { + if aut == nil { + aut = alwaysMatchAutomaton + } + + i.f = f + i.startKeyInclusive = startKeyInclusive + i.endKeyExclusive = endKeyExclusive + i.aut = aut + + return i.pointTo(startKeyInclusive) +} + +// pointTo attempts to point us to the specified location +func (i *FSTIterator) pointTo(key []byte) error { + + // tried to seek before start + if bytes.Compare(key, i.startKeyInclusive) < 0 { + key = i.startKeyInclusive + } + + // trid to see past end + if i.endKeyExclusive != nil && bytes.Compare(key, i.endKeyExclusive) > 0 { + key = i.endKeyExclusive + } + + // reset any state, pointTo always starts over + i.statesStack = i.statesStack[:0] + i.keysStack = i.keysStack[:0] + i.keysPosStack = i.keysPosStack[:0] + i.valsStack = i.valsStack[:0] + i.autStatesStack = i.autStatesStack[:0] + + root, err := i.f.decoder.stateAt(i.f.decoder.getRoot(), nil) + if err != nil { + return err + } + + autStart := i.aut.Start() + + maxQ := -1 + // root is always part of the path + i.statesStack = append(i.statesStack, root) + i.autStatesStack = append(i.autStatesStack, autStart) + for j := 0; j < len(key); j++ { + curr := i.statesStack[len(i.statesStack)-1] + autCurr := i.autStatesStack[len(i.autStatesStack)-1] + + pos, nextAddr, nextVal := curr.TransitionFor(key[j]) + if nextAddr == noneAddr { + // needed transition doesn't exist + // find last trans before the one we needed + for q := 0; q < curr.NumTransitions(); q++ { + if curr.TransitionAt(q) < key[j] { + maxQ = q + } + } + break + } + autNext := i.aut.Accept(autCurr, key[j]) + + next, err := i.f.decoder.stateAt(nextAddr, nil) + if err != nil { + return err + } + + i.statesStack = append(i.statesStack, next) + i.keysStack = append(i.keysStack, key[j]) + i.keysPosStack = append(i.keysPosStack, pos) + i.valsStack = append(i.valsStack, nextVal) + i.autStatesStack = append(i.autStatesStack, autNext) + continue + } + + if !i.statesStack[len(i.statesStack)-1].Final() || !i.aut.IsMatch(i.autStatesStack[len(i.autStatesStack)-1]) || bytes.Compare(i.keysStack, key) < 0 { + return i.next(maxQ) + } + + return nil +} + +// Current returns the key and value currently pointed to by the iterator. +// If the iterator is not pointing at a valid value (because Iterator/Next/Seek) +// returned an error previously, it may return nil,0. +func (i *FSTIterator) Current() ([]byte, uint64) { + curr := i.statesStack[len(i.statesStack)-1] + if curr.Final() { + var total uint64 + for _, v := range i.valsStack { + total += v + } + total += curr.FinalOutput() + return i.keysStack, total + } + return nil, 0 +} + +// Next advances this iterator to the next key/value pair. If there is none +// or the advancement goes beyond the configured endKeyExclusive, then +// ErrIteratorDone is returned. +func (i *FSTIterator) Next() error { + return i.next(-1) +} + +func (i *FSTIterator) next(lastOffset int) error { + + // remember where we started + if cap(i.nextStart) < len(i.keysStack) { + i.nextStart = make([]byte, len(i.keysStack)) + } else { + i.nextStart = i.nextStart[0:len(i.keysStack)] + } + copy(i.nextStart, i.keysStack) + + for true { + curr := i.statesStack[len(i.statesStack)-1] + autCurr := i.autStatesStack[len(i.autStatesStack)-1] + + if curr.Final() && i.aut.IsMatch(autCurr) && + bytes.Compare(i.keysStack, i.nextStart) > 0 { + // in final state greater than start key + return nil + } + + nextOffset := lastOffset + 1 + if nextOffset < curr.NumTransitions() { + t := curr.TransitionAt(nextOffset) + autNext := i.aut.Accept(autCurr, t) + if i.aut.CanMatch(autNext) { + pos, nextAddr, v := curr.TransitionFor(t) + + // the next slot in the statesStack might have an + // fstState instance that we can reuse + var nextPrealloc fstState + if len(i.statesStack) < cap(i.statesStack) { + nextPrealloc = i.statesStack[0:cap(i.statesStack)][len(i.statesStack)] + } + + // push onto stack + next, err := i.f.decoder.stateAt(nextAddr, nextPrealloc) + if err != nil { + return err + } + i.statesStack = append(i.statesStack, next) + i.keysStack = append(i.keysStack, t) + i.keysPosStack = append(i.keysPosStack, pos) + i.valsStack = append(i.valsStack, v) + i.autStatesStack = append(i.autStatesStack, autNext) + lastOffset = -1 + + // check to see if new keystack might have gone too far + if i.endKeyExclusive != nil && bytes.Compare(i.keysStack, i.endKeyExclusive) >= 0 { + return ErrIteratorDone + } + } else { + lastOffset = nextOffset + } + + continue + } + + if len(i.statesStack) > 1 { + // no transitions, and still room to pop + i.statesStack = i.statesStack[:len(i.statesStack)-1] + i.keysStack = i.keysStack[:len(i.keysStack)-1] + lastOffset = i.keysPosStack[len(i.keysPosStack)-1] + + i.keysPosStack = i.keysPosStack[:len(i.keysPosStack)-1] + i.valsStack = i.valsStack[:len(i.valsStack)-1] + i.autStatesStack = i.autStatesStack[:len(i.autStatesStack)-1] + continue + } else { + // stack len is 1 (root), can't go back further, we're done + break + } + + } + + return ErrIteratorDone +} + +// Seek advances this iterator to the specified key/value pair. If this key +// is not in the FST, Current() will return the next largest key. If this +// seek operation would go past the last key, or outside the configured +// startKeyInclusive/endKeyExclusive then ErrIteratorDone is returned. +func (i *FSTIterator) Seek(key []byte) error { + err := i.pointTo(key) + if err != nil { + return err + } + return nil +} + +// Close will free any resources held by this iterator. +func (i *FSTIterator) Close() error { + // at the moment we don't do anything, but wanted this for API completeness + return nil +} diff --git a/vendor/github.com/couchbase/vellum/levenshtein/dfa.go b/vendor/github.com/couchbase/vellum/levenshtein/dfa.go new file mode 100644 index 00000000..5f94a19d --- /dev/null +++ b/vendor/github.com/couchbase/vellum/levenshtein/dfa.go @@ -0,0 +1,206 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package levenshtein + +import ( + "encoding/binary" + "fmt" + "unicode" + + "github.com/couchbase/vellum/utf8" +) + +type dfa struct { + states statesStack +} + +type state struct { + next []int + match bool +} + +func (s *state) String() string { + rv := " |" + for i := 0; i < 16; i++ { + rv += fmt.Sprintf("% 5x", i) + } + rv += "\n" + for i := 0; i < len(s.next); i++ { + if i%16 == 0 { + rv += fmt.Sprintf("%x |", i/16) + } + if s.next[i] != 0 { + rv += fmt.Sprintf("% 5d", s.next[i]) + } else { + rv += " -" + } + if i%16 == 15 { + rv += "\n" + } + } + return rv +} + +type dfaBuilder struct { + dfa *dfa + lev *dynamicLevenshtein + cache map[string]int + keyBuf []byte +} + +func newDfaBuilder(lev *dynamicLevenshtein) *dfaBuilder { + dfab := &dfaBuilder{ + dfa: &dfa{ + states: make([]*state, 0, 16), + }, + lev: lev, + cache: make(map[string]int, 1024), + } + dfab.newState(false) // create state 0, invalid + return dfab +} + +func (b *dfaBuilder) build() (*dfa, error) { + var stack intsStack + stack = stack.Push(b.lev.start()) + seen := make(map[int]struct{}) + + var levState []int + stack, levState = stack.Pop() + for levState != nil { + dfaSi := b.cachedState(levState) + mmToSi, mmMismatchState, err := b.addMismatchUtf8States(dfaSi, levState) + if err != nil { + return nil, err + } + if mmToSi != 0 { + if _, ok := seen[mmToSi]; !ok { + seen[mmToSi] = struct{}{} + stack = stack.Push(mmMismatchState) + } + } + + i := 0 + for _, r := range b.lev.query { + if uint(levState[i]) > b.lev.distance { + i++ + continue + } + levNext := b.lev.accept(levState, &r) + nextSi := b.cachedState(levNext) + if nextSi != 0 { + err = b.addUtf8Sequences(true, dfaSi, nextSi, r, r) + if err != nil { + return nil, err + } + if _, ok := seen[nextSi]; !ok { + seen[nextSi] = struct{}{} + stack = stack.Push(levNext) + } + } + i++ + } + + if len(b.dfa.states) > StateLimit { + return nil, ErrTooManyStates + } + + stack, levState = stack.Pop() + } + + return b.dfa, nil +} + +func (b *dfaBuilder) cachedState(levState []int) int { + rv, _ := b.cached(levState) + return rv +} + +func levStateKey(levState []int, buf []byte) []byte { + if cap(buf) < 8*len(levState) { + buf = make([]byte, 8*len(levState)) + } else { + buf = buf[0 : 8*len(levState)] + } + for i, state := range levState { + binary.LittleEndian.PutUint64(buf[i*8:], uint64(state)) + } + return buf +} + +func (b *dfaBuilder) cached(levState []int) (int, bool) { + if !b.lev.canMatch(levState) { + return 0, true + } + b.keyBuf = levStateKey(levState, b.keyBuf) + v, ok := b.cache[string(b.keyBuf)] + if ok { + return v, true + } + match := b.lev.isMatch(levState) + b.dfa.states = b.dfa.states.Push(&state{ + next: make([]int, 256), + match: match, + }) + newV := len(b.dfa.states) - 1 + b.cache[string(b.keyBuf)] = newV + return newV, false +} + +func (b *dfaBuilder) addMismatchUtf8States(fromSi int, levState []int) (int, []int, error) { + mmState := b.lev.accept(levState, nil) + toSi, _ := b.cached(mmState) + if toSi == 0 { + return 0, nil, nil + } + err := b.addUtf8Sequences(false, fromSi, toSi, 0, unicode.MaxRune) + if err != nil { + return 0, nil, err + } + return toSi, mmState, nil +} + +func (b *dfaBuilder) addUtf8Sequences(overwrite bool, fromSi, toSi int, fromChar, toChar rune) error { + sequences, err := utf8.NewSequences(fromChar, toChar) + if err != nil { + return err + } + for _, seq := range sequences { + fsi := fromSi + for _, utf8r := range seq[:len(seq)-1] { + tsi := b.newState(false) + b.addUtf8Range(overwrite, fsi, tsi, utf8r) + fsi = tsi + } + b.addUtf8Range(overwrite, fsi, toSi, seq[len(seq)-1]) + } + return nil +} + +func (b *dfaBuilder) addUtf8Range(overwrite bool, from, to int, rang *utf8.Range) { + for by := rang.Start; by <= rang.End; by++ { + if overwrite || b.dfa.states[from].next[by] == 0 { + b.dfa.states[from].next[by] = to + } + } +} + +func (b *dfaBuilder) newState(match bool) int { + b.dfa.states = append(b.dfa.states, &state{ + next: make([]int, 256), + match: match, + }) + return len(b.dfa.states) - 1 +} diff --git a/vendor/github.com/couchbase/vellum/levenshtein/levenshtein.go b/vendor/github.com/couchbase/vellum/levenshtein/levenshtein.go new file mode 100644 index 00000000..5d1f65d1 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/levenshtein/levenshtein.go @@ -0,0 +1,90 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package levenshtein + +import ( + "fmt" +) + +// StateLimit is the maximum number of states allowed +const StateLimit = 10000 + +// ErrTooManyStates is returned if you attempt to build a Levenshtein +// automaton which requries too many states. +var ErrTooManyStates = fmt.Errorf("dfa contains more than %d states", StateLimit) + +// Levenshtein implements the vellum.Automaton interface for matching +// terms within the specified Levenshtein edit-distance of the queried +// term. This automaton recognizes utf-8 encoded bytes and computes +// the edit distance on the result code-points, not on the raw bytes. +type Levenshtein struct { + prog *dynamicLevenshtein + dfa *dfa +} + +// New creates a new Levenshtein automaton for the specified +// query string and edit distance. +func New(query string, distance int) (*Levenshtein, error) { + lev := &dynamicLevenshtein{ + query: query, + distance: uint(distance), + } + dfabuilder := newDfaBuilder(lev) + dfa, err := dfabuilder.build() + if err != nil { + return nil, err + } + return &Levenshtein{ + prog: lev, + dfa: dfa, + }, nil +} + +// Start returns the start state of this automaton. +func (l *Levenshtein) Start() int { + return 1 +} + +// IsMatch returns if the specified state is a matching state. +func (l *Levenshtein) IsMatch(s int) bool { + if s < len(l.dfa.states) { + return l.dfa.states[s].match + } + return false +} + +// CanMatch returns if the specified state can ever transition to a matching +// state. +func (l *Levenshtein) CanMatch(s int) bool { + if s < len(l.dfa.states) && s > 0 { + return true + } + return false +} + +// WillAlwaysMatch returns if the specified state will always end in a +// matching state. +func (l *Levenshtein) WillAlwaysMatch(s int) bool { + return false +} + +// Accept returns the new state, resulting from the transite byte b +// when currently in the state s. +func (l *Levenshtein) Accept(s int, b byte) int { + if s < len(l.dfa.states) { + return l.dfa.states[s].next[b] + } + return 0 +} diff --git a/vendor/github.com/couchbase/vellum/levenshtein/rune.go b/vendor/github.com/couchbase/vellum/levenshtein/rune.go new file mode 100644 index 00000000..0fefa776 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/levenshtein/rune.go @@ -0,0 +1,78 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package levenshtein + +import "unicode/utf8" + +// dynamicLevenshtein is the rune-based automaton, which is used +// during the building of the ut8-aware byte-based automaton +type dynamicLevenshtein struct { + query string + distance uint +} + +func (d *dynamicLevenshtein) start() []int { + runeCount := utf8.RuneCountInString(d.query) + rv := make([]int, runeCount+1) + for i := 0; i < runeCount+1; i++ { + rv[i] = i + } + return rv +} + +func (d *dynamicLevenshtein) isMatch(state []int) bool { + last := state[len(state)-1] + if uint(last) <= d.distance { + return true + } + return false +} + +func (d *dynamicLevenshtein) canMatch(state []int) bool { + if len(state) > 0 { + min := state[0] + for i := 1; i < len(state); i++ { + if state[i] < min { + min = state[i] + } + } + if uint(min) <= d.distance { + return true + } + } + return false +} + +func (d *dynamicLevenshtein) accept(state []int, r *rune) []int { + next := []int{state[0] + 1} + i := 0 + for _, c := range d.query { + var cost int + if r == nil || c != *r { + cost = 1 + } + v := min(min(next[i]+1, state[i+1]+1), state[i]+cost) + next = append(next, min(v, int(d.distance)+1)) + i++ + } + return next +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/vendor/github.com/couchbase/vellum/levenshtein/stack.go b/vendor/github.com/couchbase/vellum/levenshtein/stack.go new file mode 100644 index 00000000..d42f6018 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/levenshtein/stack.go @@ -0,0 +1,49 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package levenshtein + +import "fmt" + +type statesStack []*state + +func (s statesStack) String() string { + rv := "" + for i := 0; i < len(s); i++ { + matchStr := "" + if s[i].match { + matchStr = " (MATCH) " + } + rv += fmt.Sprintf("state %d%s:\n%v\n", i, matchStr, s[i]) + } + return rv +} + +func (s statesStack) Push(v *state) statesStack { + return append(s, v) +} + +type intsStack [][]int + +func (s intsStack) Push(v []int) intsStack { + return append(s, v) +} + +func (s intsStack) Pop() (intsStack, []int) { + l := len(s) + if l < 1 { + return s, nil + } + return s[:l-1], s[l-1] +} diff --git a/vendor/github.com/couchbase/vellum/merge_iterator.go b/vendor/github.com/couchbase/vellum/merge_iterator.go new file mode 100644 index 00000000..f00f7783 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/merge_iterator.go @@ -0,0 +1,188 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "bytes" +) + +// MergeFunc is used to choose the new value for a key when merging a slice +// of iterators, and the same key is observed with multiple values. +// Values presented to the MergeFunc will be in the same order as the +// original slice creating the MergeIterator. This allows some MergeFunc +// implementations to prioritize one iterator over another. +type MergeFunc func([]uint64) uint64 + +// MergeIterator implements the Iterator interface by traversing a slice +// of iterators and merging the contents of them. If the same key exists +// in mulitipe underlying iterators, a user-provided MergeFunc will be +// invoked to choose the new value. +type MergeIterator struct { + itrs []Iterator + f MergeFunc + currKs [][]byte + currVs []uint64 + + lowK []byte + lowV uint64 + lowIdxs []int + + mergeV []uint64 +} + +// NewMergeIterator creates a new MergeIterator over the provided slice of +// Iterators and with the specified MergeFunc to resolve duplicate keys. +func NewMergeIterator(itrs []Iterator, f MergeFunc) (*MergeIterator, error) { + rv := &MergeIterator{ + itrs: itrs, + f: f, + currKs: make([][]byte, len(itrs)), + currVs: make([]uint64, len(itrs)), + lowIdxs: make([]int, 0, len(itrs)), + mergeV: make([]uint64, 0, len(itrs)), + } + rv.init() + if rv.lowK == nil { + return rv, ErrIteratorDone + } + return rv, nil +} + +func (m *MergeIterator) init() { + for i, itr := range m.itrs { + m.currKs[i], m.currVs[i] = itr.Current() + } + m.updateMatches() +} + +func (m *MergeIterator) updateMatches() { + if len(m.itrs) < 1 { + return + } + m.lowK = m.currKs[0] + m.lowIdxs = m.lowIdxs[:0] + m.lowIdxs = append(m.lowIdxs, 0) + for i := 1; i < len(m.itrs); i++ { + if m.currKs[i] == nil { + continue + } + cmp := bytes.Compare(m.currKs[i], m.lowK) + if m.lowK == nil || cmp < 0 { + // reached a new low + m.lowK = m.currKs[i] + m.lowIdxs = m.lowIdxs[:0] + m.lowIdxs = append(m.lowIdxs, i) + } else if cmp == 0 { + m.lowIdxs = append(m.lowIdxs, i) + } + } + if len(m.lowIdxs) > 1 { + // merge multiple values + m.mergeV = m.mergeV[:0] + for _, vi := range m.lowIdxs { + m.mergeV = append(m.mergeV, m.currVs[vi]) + } + m.lowV = m.f(m.mergeV) + } else if len(m.lowIdxs) == 1 { + m.lowV = m.currVs[m.lowIdxs[0]] + } +} + +// Current returns the key and value currently pointed to by this iterator. +// If the iterator is not pointing at a valid value (because Iterator/Next/Seek) +// returned an error previously, it may return nil,0. +func (m *MergeIterator) Current() ([]byte, uint64) { + return m.lowK, m.lowV +} + +// Next advances this iterator to the next key/value pair. If there is none, +// then ErrIteratorDone is returned. +func (m *MergeIterator) Next() error { + // move all the current low iterators to next + for _, vi := range m.lowIdxs { + err := m.itrs[vi].Next() + if err != nil && err != ErrIteratorDone { + return err + } + m.currKs[vi], m.currVs[vi] = m.itrs[vi].Current() + } + m.updateMatches() + if m.lowK == nil { + return ErrIteratorDone + } + return nil +} + +// Seek advances this iterator to the specified key/value pair. If this key +// is not in the FST, Current() will return the next largest key. If this +// seek operation would go past the last key, then ErrIteratorDone is returned. +func (m *MergeIterator) Seek(key []byte) error { + for i := range m.itrs { + err := m.itrs[i].Seek(key) + if err != nil && err != ErrIteratorDone { + return err + } + } + m.updateMatches() + if m.lowK == nil { + return ErrIteratorDone + } + return nil +} + +// Close will attempt to close all the underlying Iterators. If any errors +// are encountered, the first will be returned. +func (m *MergeIterator) Close() error { + var rv error + for i := range m.itrs { + // close all iterators, return first error if any + err := m.itrs[i].Close() + if rv == nil { + rv = err + } + } + return rv +} + +// MergeMin chooses the minimum value +func MergeMin(vals []uint64) uint64 { + rv := vals[0] + for _, v := range vals[1:] { + if v < rv { + rv = v + } + } + return rv +} + +// MergeMax chooses the maximum value +func MergeMax(vals []uint64) uint64 { + rv := vals[0] + for _, v := range vals[1:] { + if v > rv { + rv = v + } + } + return rv +} + +// MergeSum sums the values +func MergeSum(vals []uint64) uint64 { + rv := vals[0] + for _, v := range vals[1:] { + rv += v + } + return rv +} diff --git a/vendor/github.com/couchbase/vellum/pack.go b/vendor/github.com/couchbase/vellum/pack.go new file mode 100644 index 00000000..78f3dcd5 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/pack.go @@ -0,0 +1,55 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +func deltaAddr(base, trans uint64) uint64 { + // transition dest of 0 is special case + if trans == 0 { + return 0 + } + return base - trans +} + +const packOutMask = 1<<4 - 1 + +func encodePackSize(transSize, outSize int) byte { + var rv byte + rv = byte(transSize << 4) + rv |= byte(outSize) + return rv +} + +func decodePackSize(pack byte) (transSize int, packSize int) { + transSize = int(pack >> 4) + packSize = int(pack & packOutMask) + return +} + +const maxNumTrans = 1<<6 - 1 + +func encodeNumTrans(n int) byte { + if n <= maxNumTrans { + return byte(n) + } + return 0 +} + +func readPackedUint(data []byte) (rv uint64) { + for i := range data { + shifted := uint64(data[i]) << uint(i*8) + rv |= shifted + } + return +} diff --git a/vendor/github.com/couchbase/vellum/regexp/compile.go b/vendor/github.com/couchbase/vellum/regexp/compile.go new file mode 100644 index 00000000..6922b749 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/regexp/compile.go @@ -0,0 +1,316 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "regexp/syntax" + "unicode" + + "github.com/couchbase/vellum/utf8" +) + +type compiler struct { + sizeLimit uint + insts prog +} + +func newCompiler(sizeLimit uint) *compiler { + return &compiler{ + sizeLimit: sizeLimit, + } +} + +func (c *compiler) compile(ast *syntax.Regexp) (prog, error) { + err := c.c(ast) + if err != nil { + return nil, err + } + c.insts = append(c.insts, &inst{ + op: OpMatch, + }) + return c.insts, nil +} + +func (c *compiler) c(ast *syntax.Regexp) error { + if ast.Flags&syntax.NonGreedy > 1 { + return ErrNoLazy + } + + switch ast.Op { + case syntax.OpEndLine, syntax.OpBeginLine, + syntax.OpBeginText, syntax.OpEndText: + return ErrNoEmpty + case syntax.OpWordBoundary, syntax.OpNoWordBoundary: + return ErrNoWordBoundary + case syntax.OpEmptyMatch: + return nil + case syntax.OpLiteral: + for _, r := range ast.Rune { + if ast.Flags&syntax.FoldCase > 0 { + next := syntax.Regexp{ + Op: syntax.OpCharClass, + Flags: ast.Flags & syntax.FoldCase, + Rune0: [2]rune{r, r}, + } + next.Rune = next.Rune0[0:2] + return c.c(&next) + } + seqs, err := utf8.NewSequences(r, r) + if err != nil { + return err + } + for _, seq := range seqs { + c.compileUtf8Ranges(seq) + } + } + case syntax.OpAnyChar: + next := syntax.Regexp{ + Op: syntax.OpCharClass, + Flags: ast.Flags & syntax.FoldCase, + Rune0: [2]rune{0, unicode.MaxRune}, + } + next.Rune = next.Rune0[:2] + return c.c(&next) + case syntax.OpAnyCharNotNL: + next := syntax.Regexp{ + Op: syntax.OpCharClass, + Flags: ast.Flags & syntax.FoldCase, + Rune: []rune{0, 0x09, 0x0B, unicode.MaxRune}, + } + return c.c(&next) + case syntax.OpCharClass: + return c.compileClass(ast) + case syntax.OpCapture: + return c.c(ast.Sub[0]) + case syntax.OpConcat: + for _, sub := range ast.Sub { + err := c.c(sub) + if err != nil { + return err + } + } + return nil + case syntax.OpAlternate: + if len(ast.Sub) == 0 { + return nil + } + jmpsToEnd := []uint{} + + // does not handle last entry + for i := 0; i < len(ast.Sub)-1; i++ { + sub := ast.Sub[i] + split := c.emptySplit() + j1 := c.top() + err := c.c(sub) + if err != nil { + return err + } + jmpsToEnd = append(jmpsToEnd, c.emptyJump()) + j2 := c.top() + c.setSplit(split, j1, j2) + } + // handle last entry + err := c.c(ast.Sub[len(ast.Sub)-1]) + if err != nil { + return err + } + end := uint(len(c.insts)) + for _, jmpToEnd := range jmpsToEnd { + c.setJump(jmpToEnd, end) + } + case syntax.OpQuest: + split := c.emptySplit() + j1 := c.top() + err := c.c(ast.Sub[0]) + if err != nil { + return err + } + j2 := c.top() + c.setSplit(split, j1, j2) + + case syntax.OpStar: + j1 := c.top() + split := c.emptySplit() + j2 := c.top() + err := c.c(ast.Sub[0]) + if err != nil { + return err + } + jmp := c.emptyJump() + j3 := uint(len(c.insts)) + + c.setJump(jmp, j1) + c.setSplit(split, j2, j3) + + case syntax.OpPlus: + j1 := c.top() + err := c.c(ast.Sub[0]) + if err != nil { + return err + } + split := c.emptySplit() + j2 := c.top() + c.setSplit(split, j1, j2) + + case syntax.OpRepeat: + if ast.Max == -1 { + for i := 0; i < ast.Min; i++ { + err := c.c(ast.Sub[0]) + if err != nil { + return err + } + } + next := syntax.Regexp{ + Op: syntax.OpStar, + Flags: ast.Flags, + Sub: ast.Sub, + Sub0: ast.Sub0, + Rune: ast.Rune, + Rune0: ast.Rune0, + } + return c.c(&next) + } + for i := 0; i < ast.Min; i++ { + err := c.c(ast.Sub[0]) + if err != nil { + return err + } + } + var splits, starts []uint + for i := ast.Min; i < ast.Max; i++ { + splits = append(splits, c.emptySplit()) + starts = append(starts, uint(len(c.insts))) + err := c.c(ast.Sub[0]) + if err != nil { + return err + } + } + end := uint(len(c.insts)) + for i := 0; i < len(splits); i++ { + c.setSplit(splits[i], starts[i], end) + } + + } + + return c.checkSize() +} + +func (c *compiler) checkSize() error { + if uint(len(c.insts)*instSize) > c.sizeLimit { + return ErrCompiledTooBig + } + return nil +} + +func (c *compiler) compileClass(ast *syntax.Regexp) error { + if len(ast.Rune) == 0 { + return nil + } + var jmps []uint + + // does not do last pair + for i := 0; i < len(ast.Rune)-2; i += 2 { + rstart := ast.Rune[i] + rend := ast.Rune[i+1] + + split := c.emptySplit() + j1 := c.top() + err := c.compileClassRange(rstart, rend) + if err != nil { + return err + } + jmps = append(jmps, c.emptyJump()) + j2 := c.top() + c.setSplit(split, j1, j2) + } + // handle last pair + rstart := ast.Rune[len(ast.Rune)-2] + rend := ast.Rune[len(ast.Rune)-1] + err := c.compileClassRange(rstart, rend) + if err != nil { + return err + } + end := c.top() + for _, jmp := range jmps { + c.setJump(jmp, end) + } + return nil +} + +func (c *compiler) compileClassRange(startR, endR rune) error { + seqs, err := utf8.NewSequences(startR, endR) + if err != nil { + return err + } + var jmps []uint + + // does not do last entry + for i := 0; i < len(seqs)-1; i++ { + seq := seqs[i] + split := c.emptySplit() + j1 := c.top() + c.compileUtf8Ranges(seq) + jmps = append(jmps, c.emptyJump()) + j2 := c.top() + c.setSplit(split, j1, j2) + } + // handle last entry + c.compileUtf8Ranges(seqs[len(seqs)-1]) + end := c.top() + for _, jmp := range jmps { + c.setJump(jmp, end) + } + + return nil +} + +func (c *compiler) compileUtf8Ranges(seq utf8.Sequence) { + for _, r := range seq { + c.insts = append(c.insts, &inst{ + op: OpRange, + rangeStart: r.Start, + rangeEnd: r.End, + }) + } +} + +func (c *compiler) emptySplit() uint { + c.insts = append(c.insts, &inst{ + op: OpSplit, + }) + return c.top() - 1 +} + +func (c *compiler) emptyJump() uint { + c.insts = append(c.insts, &inst{ + op: OpJmp, + }) + return c.top() - 1 +} + +func (c *compiler) setSplit(i, pc1, pc2 uint) { + split := c.insts[i] + split.splitA = pc1 + split.splitB = pc2 +} + +func (c *compiler) setJump(i, pc uint) { + jmp := c.insts[i] + jmp.to = pc +} + +func (c *compiler) top() uint { + return uint(len(c.insts)) +} diff --git a/vendor/github.com/couchbase/vellum/regexp/dfa.go b/vendor/github.com/couchbase/vellum/regexp/dfa.go new file mode 100644 index 00000000..9864606b --- /dev/null +++ b/vendor/github.com/couchbase/vellum/regexp/dfa.go @@ -0,0 +1,188 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "encoding/binary" + "fmt" +) + +// StateLimit is the maximum number of states allowed +const StateLimit = 10000 + +// ErrTooManyStates is returned if you attempt to build a Levenshtein +// automaton which requries too many states. +var ErrTooManyStates = fmt.Errorf("dfa contains more than %d states", + StateLimit) + +type dfaBuilder struct { + dfa *dfa + cache map[string]int + keyBuf []byte +} + +func newDfaBuilder(insts prog) *dfaBuilder { + d := &dfaBuilder{ + dfa: &dfa{ + insts: insts, + states: make([]*state, 0, 16), + }, + cache: make(map[string]int, 1024), + } + // add 0 state that is invalid + d.dfa.states = append(d.dfa.states, &state{ + next: make([]int, 256), + match: false, + }) + return d +} + +func (d *dfaBuilder) build() (*dfa, error) { + cur := newSparseSet(uint(len(d.dfa.insts))) + next := newSparseSet(uint(len(d.dfa.insts))) + + d.dfa.add(cur, 0) + states := intStack{d.cachedState(cur)} + seen := make(map[int]struct{}) + var s int + states, s = states.Pop() + for s != 0 { + for b := 0; b < 256; b++ { + ns := d.runState(cur, next, s, byte(b)) + if ns != 0 { + if _, ok := seen[ns]; !ok { + seen[ns] = struct{}{} + states = states.Push(ns) + } + } + if len(d.dfa.states) > StateLimit { + return nil, ErrTooManyStates + } + } + states, s = states.Pop() + } + return d.dfa, nil +} + +func (d *dfaBuilder) runState(cur, next *sparseSet, state int, b byte) int { + cur.Clear() + for _, ip := range d.dfa.states[state].insts { + cur.Add(ip) + } + d.dfa.run(cur, next, b) + nextState := d.cachedState(next) + d.dfa.states[state].next[b] = nextState + return nextState +} + +func instsKey(insts []uint, buf []byte) []byte { + if cap(buf) < 8*len(insts) { + buf = make([]byte, 8*len(insts)) + } else { + buf = buf[0 : 8*len(insts)] + } + for i, inst := range insts { + binary.LittleEndian.PutUint64(buf[i*8:], uint64(inst)) + } + return buf +} + +func (d *dfaBuilder) cachedState(set *sparseSet) int { + var insts []uint + var isMatch bool + for i := uint(0); i < uint(set.Len()); i++ { + ip := set.Get(i) + switch d.dfa.insts[ip].op { + case OpRange: + insts = append(insts, ip) + case OpMatch: + isMatch = true + insts = append(insts, ip) + } + } + if len(insts) == 0 { + return 0 + } + d.keyBuf = instsKey(insts, d.keyBuf) + v, ok := d.cache[string(d.keyBuf)] + if ok { + return v + } + d.dfa.states = append(d.dfa.states, &state{ + insts: insts, + next: make([]int, 256), + match: isMatch, + }) + newV := len(d.dfa.states) - 1 + d.cache[string(d.keyBuf)] = newV + return newV +} + +type dfa struct { + insts prog + states []*state +} + +func (d *dfa) add(set *sparseSet, ip uint) { + if set.Contains(ip) { + return + } + set.Add(ip) + switch d.insts[ip].op { + case OpJmp: + d.add(set, d.insts[ip].to) + case OpSplit: + d.add(set, d.insts[ip].splitA) + d.add(set, d.insts[ip].splitB) + } +} + +func (d *dfa) run(from, to *sparseSet, b byte) bool { + to.Clear() + var isMatch bool + for i := uint(0); i < uint(from.Len()); i++ { + ip := from.Get(i) + switch d.insts[ip].op { + case OpMatch: + isMatch = true + case OpRange: + if d.insts[ip].rangeStart <= b && + b <= d.insts[ip].rangeEnd { + d.add(to, ip+1) + } + } + } + return isMatch +} + +type state struct { + insts []uint + next []int + match bool +} + +type intStack []int + +func (s intStack) Push(v int) intStack { + return append(s, v) +} + +func (s intStack) Pop() (intStack, int) { + l := len(s) + if l < 1 { + return s, 0 + } + return s[:l-1], s[l-1] +} diff --git a/vendor/github.com/couchbase/vellum/regexp/inst.go b/vendor/github.com/couchbase/vellum/regexp/inst.go new file mode 100644 index 00000000..61cbf2f3 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/regexp/inst.go @@ -0,0 +1,62 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import "fmt" + +// instOp represents a instruction operation +type instOp int + +// the enumeration of operations +const ( + OpMatch instOp = iota + OpJmp + OpSplit + OpRange +) + +// instSize is the approxmiate size of the an inst struct in bytes +const instSize = 40 + +type inst struct { + op instOp + to uint + splitA uint + splitB uint + rangeStart byte + rangeEnd byte +} + +func (i *inst) String() string { + switch i.op { + case OpJmp: + return fmt.Sprintf("JMP: %d", i.to) + case OpSplit: + return fmt.Sprintf("SPLIT: %d - %d", i.splitA, i.splitB) + case OpRange: + return fmt.Sprintf("RANGE: %x - %x", i.rangeStart, i.rangeEnd) + } + return "MATCH" +} + +type prog []*inst + +func (p prog) String() string { + rv := "\n" + for i, pi := range p { + rv += fmt.Sprintf("%d %v\n", i, pi) + } + return rv +} diff --git a/vendor/github.com/couchbase/vellum/regexp/regexp.go b/vendor/github.com/couchbase/vellum/regexp/regexp.go new file mode 100644 index 00000000..ed0e7823 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/regexp/regexp.go @@ -0,0 +1,113 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "fmt" + "regexp/syntax" +) + +// ErrNoEmpty returned when "zero width assertions" are used +var ErrNoEmpty = fmt.Errorf("zero width assertions not allowed") + +// ErrNoWordBoundary returned when word boundaries are used +var ErrNoWordBoundary = fmt.Errorf("word boundaries are not allowed") + +// ErrNoBytes returned when byte literals are used +var ErrNoBytes = fmt.Errorf("byte literals are not allowed") + +// ErrNoLazy returned when lazy quantifiers are used +var ErrNoLazy = fmt.Errorf("lazy quantifiers are not allowed") + +// ErrCompiledTooBig returned when regular expression parses into +// too many instructions +var ErrCompiledTooBig = fmt.Errorf("too many instructions") + +// Regexp implements the vellum.Automaton interface for matcing a user +// specified regular expression. +type Regexp struct { + orig string + dfa *dfa +} + +// NewRegexp creates a new Regular Expression automaton with the specified +// expression. By default it is limited to approximately 10MB for the +// compiled finite state automaton. If this size is exceeded, +// ErrCompiledTooBig will be returned. +func New(expr string) (*Regexp, error) { + return NewWithLimit(expr, 10*(1<<20)) +} + +// NewRegexpWithLimit creates a new Regular Expression automaton with +// the specified expression. The size of the compiled finite state +// automaton exceeds the user specified size, ErrCompiledTooBig will be +// returned. +func NewWithLimit(expr string, size uint) (*Regexp, error) { + parsed, err := syntax.Parse(expr, syntax.Perl) + if err != nil { + return nil, err + } + compiler := newCompiler(size) + insts, err := compiler.compile(parsed) + if err != nil { + return nil, err + } + dfaBuilder := newDfaBuilder(insts) + dfa, err := dfaBuilder.build() + if err != nil { + return nil, err + } + return &Regexp{ + orig: expr, + dfa: dfa, + }, nil +} + +// Start returns the start state of this automaton. +func (r *Regexp) Start() int { + return 1 +} + +// IsMatch returns if the specified state is a matching state. +func (r *Regexp) IsMatch(s int) bool { + if s < len(r.dfa.states) { + return r.dfa.states[s].match + } + return false +} + +// CanMatch returns if the specified state can ever transition to a matching +// state. +func (r *Regexp) CanMatch(s int) bool { + if s < len(r.dfa.states) && s > 0 { + return true + } + return false +} + +// WillAlwaysMatch returns if the specified state will always end in a +// matching state. +func (r *Regexp) WillAlwaysMatch(int) bool { + return false +} + +// Accept returns the new state, resulting from the transite byte b +// when currently in the state s. +func (r *Regexp) Accept(s int, b byte) int { + if s < len(r.dfa.states) { + return r.dfa.states[s].next[b] + } + return 0 +} diff --git a/vendor/github.com/couchbase/vellum/regexp/sparse.go b/vendor/github.com/couchbase/vellum/regexp/sparse.go new file mode 100644 index 00000000..7afbfceb --- /dev/null +++ b/vendor/github.com/couchbase/vellum/regexp/sparse.go @@ -0,0 +1,54 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +type sparseSet struct { + dense []uint + sparse []uint + size uint +} + +func newSparseSet(size uint) *sparseSet { + return &sparseSet{ + dense: make([]uint, size), + sparse: make([]uint, size), + size: 0, + } +} + +func (s *sparseSet) Len() int { + return int(s.size) +} + +func (s *sparseSet) Add(ip uint) uint { + i := s.size + s.dense[i] = ip + s.sparse[ip] = i + s.size++ + return i +} + +func (s *sparseSet) Get(i uint) uint { + return s.dense[i] +} + +func (s *sparseSet) Contains(ip uint) bool { + i := s.sparse[ip] + return i < s.size && s.dense[i] == ip +} + +func (s *sparseSet) Clear() { + s.size = 0 +} diff --git a/vendor/github.com/couchbase/vellum/registry.go b/vendor/github.com/couchbase/vellum/registry.go new file mode 100644 index 00000000..3721a7c9 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/registry.go @@ -0,0 +1,116 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "hash" + "hash/fnv" +) + +type registryCell struct { + addr int + node *builderNode +} + +type registry struct { + table []registryCell + tableSize uint + mruSize uint + hasher hash.Hash64 +} + +func newRegistry(tableSize, mruSize int) *registry { + nsize := tableSize * mruSize + rv := ®istry{ + table: make([]registryCell, nsize), + tableSize: uint(tableSize), + mruSize: uint(mruSize), + hasher: fnv.New64a(), + } + return rv +} + +func (r *registry) Reset() { + for i := 0; i < len(r.table); i++ { + r.table[i] = registryCell{} + } + r.hasher.Reset() +} + +func (r *registry) entry(node *builderNode) (bool, int, *registryCell) { + if len(r.table) == 0 { + return false, 0, nil + } + bucket := r.hash(node) + start := r.mruSize * uint(bucket) + end := start + r.mruSize + rc := registryCache(r.table[start:end]) + return rc.entry(node) +} + +const fnvPrime = 1099511628211 + +func (r *registry) hash(b *builderNode) int { + var final uint64 + if b.final { + final = 1 + } + + var h uint64 = 14695981039346656037 + h = (h ^ final) * fnvPrime + h = (h ^ b.finalOutput) * fnvPrime + for _, t := range b.trans { + h = (h ^ uint64(t.in)) * fnvPrime + h = (h ^ t.out) * fnvPrime + h = (h ^ uint64(t.addr)) * fnvPrime + } + return int(h % uint64(r.tableSize)) +} + +type registryCache []registryCell + +func (r registryCache) entry(node *builderNode) (bool, int, *registryCell) { + if len(r) == 1 { + if r[0].node != nil && r[0].node.equiv(node) { + return true, r[0].addr, nil + } + r[0].node = node + return false, 0, &r[0] + } + for i := range r { + if r[i].node != nil && r[i].node.equiv(node) { + addr := r[i].addr + r.promote(i) + return true, addr, nil + } + } + // no match + last := len(r) - 1 + r[last].node = node // discard LRU + r.promote(last) + return false, 0, &r[0] + +} + +func (r registryCache) promote(i int) { + for i > 0 { + r.swap(i-1, i) + i-- + } +} + +func (r registryCache) swap(i, j int) { + r[i], r[j] = r[j], r[i] +} diff --git a/vendor/github.com/couchbase/vellum/transducer.go b/vendor/github.com/couchbase/vellum/transducer.go new file mode 100644 index 00000000..753c422d --- /dev/null +++ b/vendor/github.com/couchbase/vellum/transducer.go @@ -0,0 +1,55 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +// Transducer represents the general contract of a byte-based finite transducer +type Transducer interface { + + // all transducers are also automatons + Automaton + + // IsMatchWithValue returns true if and only if the state is a match + // additionally it returns a states final value (if any) + IsMatchWithVal(int) (bool, uint64) + + // Accept returns the next state given the input to the specified state + // additionally it returns the value associated with the transition + AcceptWithVal(int, byte) (int, uint64) +} + +// TransducerGet implements an generic Get() method which works +// on any implementation of Transducer +// The caller MUST check the boolean return value for a match. +// Zero is a valid value regardless of match status, +// and if it is NOT a match, the value collected so far is returned. +func TransducerGet(t Transducer, k []byte) (bool, uint64) { + var total uint64 + i := 0 + curr := t.Start() + for t.CanMatch(curr) && i < len(k) { + var transVal uint64 + curr, transVal = t.AcceptWithVal(curr, k[i]) + if curr == noneAddr { + break + } + total += transVal + i++ + } + if i != len(k) { + return false, total + } + match, finalVal := t.IsMatchWithVal(curr) + return match, total + finalVal +} diff --git a/vendor/github.com/couchbase/vellum/utf8/utf8.go b/vendor/github.com/couchbase/vellum/utf8/utf8.go new file mode 100644 index 00000000..47dbe9d1 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/utf8/utf8.go @@ -0,0 +1,246 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utf8 + +import ( + "fmt" + "unicode/utf8" +) + +// Sequences is a collection of Sequence +type Sequences []Sequence + +// NewSequences constructs a collection of Sequence which describe the +// byte ranges covered between the start and end runes. +func NewSequences(start, end rune) (Sequences, error) { + var rv Sequences + + var rangeStack rangeStack + rangeStack = rangeStack.Push(&scalarRange{start, end}) + + rangeStack, r := rangeStack.Pop() +TOP: + for r != nil { + INNER: + for { + r1, r2 := r.split() + if r1 != nil { + rangeStack = rangeStack.Push(&scalarRange{r2.start, r2.end}) + r.start = r1.start + r.end = r1.end + continue INNER + } + if !r.valid() { + rangeStack, r = rangeStack.Pop() + continue TOP + } + for i := 1; i < utf8.UTFMax; i++ { + max := maxScalarValue(i) + if r.start <= max && max < r.end { + rangeStack = rangeStack.Push(&scalarRange{max + 1, r.end}) + r.end = max + continue INNER + } + } + asciiRange := r.ascii() + if asciiRange != nil { + rv = append(rv, Sequence{ + asciiRange, + }) + rangeStack, r = rangeStack.Pop() + continue TOP + } + for i := uint(1); i < utf8.UTFMax; i++ { + m := rune((1 << (6 * i)) - 1) + if (r.start & ^m) != (r.end & ^m) { + if (r.start & m) != 0 { + rangeStack = rangeStack.Push(&scalarRange{(r.start | m) + 1, r.end}) + r.end = r.start | m + continue INNER + } + if (r.end & m) != m { + rangeStack = rangeStack.Push(&scalarRange{r.end & ^m, r.end}) + r.end = (r.end & ^m) - 1 + continue INNER + } + } + } + start := make([]byte, utf8.UTFMax) + end := make([]byte, utf8.UTFMax) + n, m := r.encode(start, end) + seq, err := SequenceFromEncodedRange(start[0:n], end[0:m]) + if err != nil { + return nil, err + } + rv = append(rv, seq) + rangeStack, r = rangeStack.Pop() + continue TOP + } + } + + return rv, nil +} + +// Sequence is a collection of *Range +type Sequence []*Range + +// SequenceFromEncodedRange creates sequence from the encoded bytes +func SequenceFromEncodedRange(start, end []byte) (Sequence, error) { + if len(start) != len(end) { + return nil, fmt.Errorf("byte slices must be the same length") + } + switch len(start) { + case 2: + return Sequence{ + &Range{start[0], end[0]}, + &Range{start[1], end[1]}, + }, nil + case 3: + return Sequence{ + &Range{start[0], end[0]}, + &Range{start[1], end[1]}, + &Range{start[2], end[2]}, + }, nil + case 4: + return Sequence{ + &Range{start[0], end[0]}, + &Range{start[1], end[1]}, + &Range{start[2], end[2]}, + &Range{start[3], end[3]}, + }, nil + } + + return nil, fmt.Errorf("invalid encoded byte length") +} + +// Matches checks to see if the provided byte slice matches the Sequence +func (u Sequence) Matches(bytes []byte) bool { + if len(bytes) < len(u) { + return false + } + for i := 0; i < len(u); i++ { + if !u[i].matches(bytes[i]) { + return false + } + } + return true +} + +func (u Sequence) String() string { + switch len(u) { + case 1: + return fmt.Sprintf("%v", u[0]) + case 2: + return fmt.Sprintf("%v%v", u[0], u[1]) + case 3: + return fmt.Sprintf("%v%v%v", u[0], u[1], u[2]) + case 4: + return fmt.Sprintf("%v%v%v%v", u[0], u[1], u[2], u[3]) + default: + return fmt.Sprintf("invalid utf8 sequence") + } +} + +// Range describes a single range of byte values +type Range struct { + Start byte + End byte +} + +func (u Range) matches(b byte) bool { + if u.Start <= b && b <= u.End { + return true + } + return false +} + +func (u Range) String() string { + if u.Start == u.End { + return fmt.Sprintf("[%X]", u.Start) + } + return fmt.Sprintf("[%X-%X]", u.Start, u.End) +} + +type scalarRange struct { + start rune + end rune +} + +func (s *scalarRange) String() string { + return fmt.Sprintf("ScalarRange(%d,%d)", s.start, s.end) +} + +// split this scalar range if it overlaps with a surrogate codepoint +func (s *scalarRange) split() (*scalarRange, *scalarRange) { + if s.start < 0xe000 && s.end > 0xd7ff { + return &scalarRange{ + start: s.start, + end: 0xd7ff, + }, + &scalarRange{ + start: 0xe000, + end: s.end, + } + } + return nil, nil +} + +func (s *scalarRange) valid() bool { + return s.start <= s.end +} + +func (s *scalarRange) ascii() *Range { + if s.valid() && s.end <= 0x7f { + return &Range{ + Start: byte(s.start), + End: byte(s.end), + } + } + return nil +} + +// start and end MUST have capacity for utf8.UTFMax bytes +func (s *scalarRange) encode(start, end []byte) (int, int) { + n := utf8.EncodeRune(start, s.start) + m := utf8.EncodeRune(end, s.end) + return n, m +} + +type rangeStack []*scalarRange + +func (s rangeStack) Push(v *scalarRange) rangeStack { + return append(s, v) +} + +func (s rangeStack) Pop() (rangeStack, *scalarRange) { + l := len(s) + if l < 1 { + return s, nil + } + return s[:l-1], s[l-1] +} + +func maxScalarValue(nbytes int) rune { + switch nbytes { + case 1: + return 0x007f + case 2: + return 0x07FF + case 3: + return 0xFFFF + default: + return 0x10FFFF + } +} diff --git a/vendor/github.com/couchbase/vellum/vellum.go b/vendor/github.com/couchbase/vellum/vellum.go new file mode 100644 index 00000000..b2537b3f --- /dev/null +++ b/vendor/github.com/couchbase/vellum/vellum.go @@ -0,0 +1,111 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package vellum is a library for building, serializing and executing an FST (finite +state transducer). + +There are two distinct phases, building an FST and using it. + +When building an FST, you insert keys ([]byte) and their associated value +(uint64). Insert operations MUST be done in lexicographic order. While +building the FST, data is streamed to an underlying Writer. At the conclusion +of building, you MUST call Close() on the builder. + +After completion of the build phase, you can either Open() the FST if you +serialized it to disk. Alternatively, if you already have the bytes in +memory, you can use Load(). By default, Open() will use mmap to avoid loading +the entire file into memory. + +Once the FST is ready, you can use the Contains() method to see if a keys is +in the FST. You can use the Get() method to see if a key is in the FST and +retrieve it's associated value. And, you can use the Iterator method to +enumerate key/value pairs within a specified range. + +*/ +package vellum + +import ( + "errors" + "io" +) + +// ErrOutOfOrder is returned when values are not inserted in +// lexicographic order. +var ErrOutOfOrder = errors.New("values not inserted in lexicographic order") + +// ErrIteratorDone is returned by Iterator/Next/Seek methods when the +// Current() value pointed to by the iterator is greater than the last +// key in this FST, or outside the configured startKeyInclusive/endKeyExclusive +// range of the Iterator. +var ErrIteratorDone = errors.New("iterator-done") + +// BuilderOpts is a structure to let advanced users customize the behavior +// of the builder and some aspects of the generated FST. +type BuilderOpts struct { + Encoder int + RegistryTableSize int + RegistryMRUSize int +} + +// New returns a new Builder which will stream out the +// underlying representation to the provided Writer as the set is built. +func New(w io.Writer, opts *BuilderOpts) (*Builder, error) { + return newBuilder(w, opts) +} + +// Open loads the FST stored in the provided path +func Open(path string) (*FST, error) { + return open(path) +} + +// Load will return the FST represented by the provided byte slice. +func Load(data []byte) (*FST, error) { + return new(data, nil) +} + +// Merge will iterate through the provided Iterators, merge duplicate keys +// with the provided MergeFunc, and build a new FST to the provided Writer. +func Merge(w io.Writer, opts *BuilderOpts, itrs []Iterator, f MergeFunc) error { + builder, err := New(w, opts) + if err != nil { + return err + } + + itr, err := NewMergeIterator(itrs, f) + for err == nil { + k, v := itr.Current() + err = builder.Insert(k, v) + if err != nil { + return err + } + err = itr.Next() + } + + if err != nil && err != ErrIteratorDone { + return err + } + + err = itr.Close() + if err != nil { + return err + } + + err = builder.Close() + if err != nil { + return err + } + + return nil +} diff --git a/vendor/github.com/couchbase/vellum/vellum_mmap.go b/vendor/github.com/couchbase/vellum/vellum_mmap.go new file mode 100644 index 00000000..5acd2f47 --- /dev/null +++ b/vendor/github.com/couchbase/vellum/vellum_mmap.go @@ -0,0 +1,60 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !nommap + +package vellum + +import ( + "os" + + mmap "github.com/edsrzf/mmap-go" +) + +type mmapWrapper struct { + f *os.File + mm mmap.MMap +} + +func (m *mmapWrapper) Close() (err error) { + if m.mm != nil { + err = m.mm.Unmap() + } + // try to close file even if unmap failed + if m.f != nil { + err2 := m.f.Close() + if err == nil { + // try to return first error + err = err2 + } + } + return +} + +func open(path string) (*FST, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + mm, err := mmap.Map(f, mmap.RDONLY, 0) + if err != nil { + // mmap failed, try to close the file + _ = f.Close() + return nil, err + } + return new(mm, &mmapWrapper{ + f: f, + mm: mm, + }) +} diff --git a/vendor/github.com/spf13/afero/memradix.go b/vendor/github.com/couchbase/vellum/vellum_nommap.go similarity index 64% rename from vendor/github.com/spf13/afero/memradix.go rename to vendor/github.com/couchbase/vellum/vellum_nommap.go index 87527f35..e9852728 100644 --- a/vendor/github.com/spf13/afero/memradix.go +++ b/vendor/github.com/couchbase/vellum/vellum_nommap.go @@ -1,9 +1,10 @@ -// Copyright © 2014 Steve Francia . +// Copyright (c) 2017 Couchbase, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 +// +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -11,4 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package afero +// +build nommap + +package vellum + +import "io/ioutil" + +func open(path string) (*FST, error) { + data, err := ioutil.ReadFile(string) + if err != nil { + return nil, err + } + return new(data, nil) +} diff --git a/vendor/github.com/couchbase/vellum/writer.go b/vendor/github.com/couchbase/vellum/writer.go new file mode 100644 index 00000000..d655d47f --- /dev/null +++ b/vendor/github.com/couchbase/vellum/writer.go @@ -0,0 +1,92 @@ +// Copyright (c) 2017 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package vellum + +import ( + "bufio" + "io" +) + +// A writer is a buffered writer used by vellum. It counts how many bytes have +// been written and has some convenience methods used for encoding the data. +type writer struct { + w *bufio.Writer + counter int +} + +func newWriter(w io.Writer) *writer { + return &writer{ + w: bufio.NewWriter(w), + } +} + +func (w *writer) Reset(newWriter io.Writer) { + w.w.Reset(newWriter) + w.counter = 0 +} + +func (w *writer) WriteByte(c byte) error { + err := w.w.WriteByte(c) + if err != nil { + return err + } + w.counter++ + return nil +} + +func (w *writer) Write(p []byte) (int, error) { + n, err := w.w.Write(p) + w.counter += n + return n, err +} + +func (w *writer) Flush() error { + return w.w.Flush() +} + +func (w *writer) WritePackedUintIn(v uint64, n int) error { + for shift := uint(0); shift < uint(n*8); shift += 8 { + err := w.WriteByte(byte(v >> shift)) + if err != nil { + return err + } + } + + return nil +} + +func (w *writer) WritePackedUint(v uint64) error { + n := packedSize(v) + return w.WritePackedUintIn(v, n) +} + +func packedSize(n uint64) int { + if n < 1<<8 { + return 1 + } else if n < 1<<16 { + return 2 + } else if n < 1<<24 { + return 3 + } else if n < 1<<32 { + return 4 + } else if n < 1<<40 { + return 5 + } else if n < 1<<48 { + return 6 + } else if n < 1<<56 { + return 7 + } + return 8 +} diff --git a/vendor/github.com/edsrzf/mmap-go/LICENSE b/vendor/github.com/edsrzf/mmap-go/LICENSE new file mode 100644 index 00000000..8f05f338 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2011, Evan Shaw +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/edsrzf/mmap-go/README.md b/vendor/github.com/edsrzf/mmap-go/README.md new file mode 100644 index 00000000..4cc2bfe1 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/README.md @@ -0,0 +1,12 @@ +mmap-go +======= + +mmap-go is a portable mmap package for the [Go programming language](http://golang.org). +It has been tested on Linux (386, amd64), OS X, and Windows (386). It should also +work on other Unix-like platforms, but hasn't been tested with them. I'm interested +to hear about the results. + +I haven't been able to add more features without adding significant complexity, +so mmap-go doesn't support mprotect, mincore, and maybe a few other things. +If you're running on a Unix-like platform and need some of these features, +I suggest Gustavo Niemeyer's [gommap](http://labix.org/gommap). diff --git a/vendor/github.com/edsrzf/mmap-go/mmap.go b/vendor/github.com/edsrzf/mmap-go/mmap.go new file mode 100644 index 00000000..14fb2258 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap.go @@ -0,0 +1,116 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file defines the common package interface and contains a little bit of +// factored out logic. + +// Package mmap allows mapping files into memory. It tries to provide a simple, reasonably portable interface, +// but doesn't go out of its way to abstract away every little platform detail. +// This specifically means: +// * forked processes may or may not inherit mappings +// * a file's timestamp may or may not be updated by writes through mappings +// * specifying a size larger than the file's actual size can increase the file's size +// * If the mapped file is being modified by another process while your program's running, don't expect consistent results between platforms +package mmap + +import ( + "errors" + "os" + "reflect" + "unsafe" +) + +const ( + // RDONLY maps the memory read-only. + // Attempts to write to the MMap object will result in undefined behavior. + RDONLY = 0 + // RDWR maps the memory as read-write. Writes to the MMap object will update the + // underlying file. + RDWR = 1 << iota + // COPY maps the memory as copy-on-write. Writes to the MMap object will affect + // memory, but the underlying file will remain unchanged. + COPY + // If EXEC is set, the mapped memory is marked as executable. + EXEC +) + +const ( + // If the ANON flag is set, the mapped memory will not be backed by a file. + ANON = 1 << iota +) + +// MMap represents a file mapped into memory. +type MMap []byte + +// Map maps an entire file into memory. +// If ANON is set in flags, f is ignored. +func Map(f *os.File, prot, flags int) (MMap, error) { + return MapRegion(f, -1, prot, flags, 0) +} + +// MapRegion maps part of a file into memory. +// The offset parameter must be a multiple of the system's page size. +// If length < 0, the entire file will be mapped. +// If ANON is set in flags, f is ignored. +func MapRegion(f *os.File, length int, prot, flags int, offset int64) (MMap, error) { + if offset%int64(os.Getpagesize()) != 0 { + return nil, errors.New("offset parameter must be a multiple of the system's page size") + } + + var fd uintptr + if flags&ANON == 0 { + fd = uintptr(f.Fd()) + if length < 0 { + fi, err := f.Stat() + if err != nil { + return nil, err + } + length = int(fi.Size()) + } + } else { + if length <= 0 { + return nil, errors.New("anonymous mapping requires non-zero length") + } + fd = ^uintptr(0) + } + return mmap(length, uintptr(prot), uintptr(flags), fd, offset) +} + +func (m *MMap) header() *reflect.SliceHeader { + return (*reflect.SliceHeader)(unsafe.Pointer(m)) +} + +// Lock keeps the mapped region in physical memory, ensuring that it will not be +// swapped out. +func (m MMap) Lock() error { + dh := m.header() + return lock(dh.Data, uintptr(dh.Len)) +} + +// Unlock reverses the effect of Lock, allowing the mapped region to potentially +// be swapped out. +// If m is already unlocked, aan error will result. +func (m MMap) Unlock() error { + dh := m.header() + return unlock(dh.Data, uintptr(dh.Len)) +} + +// Flush synchronizes the mapping's contents to the file's contents on disk. +func (m MMap) Flush() error { + dh := m.header() + return flush(dh.Data, uintptr(dh.Len)) +} + +// Unmap deletes the memory mapped region, flushes any remaining changes, and sets +// m to nil. +// Trying to read or write any remaining references to m after Unmap is called will +// result in undefined behavior. +// Unmap should only be called on the slice value that was originally returned from +// a call to Map. Calling Unmap on a derived slice may cause errors. +func (m *MMap) Unmap() error { + dh := m.header() + err := unmap(dh.Data, uintptr(dh.Len)) + *m = nil + return err +} diff --git a/vendor/github.com/edsrzf/mmap-go/mmap_unix.go b/vendor/github.com/edsrzf/mmap-go/mmap_unix.go new file mode 100644 index 00000000..4af98420 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap_unix.go @@ -0,0 +1,67 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux openbsd solaris netbsd + +package mmap + +import ( + "syscall" +) + +func mmap(len int, inprot, inflags, fd uintptr, off int64) ([]byte, error) { + flags := syscall.MAP_SHARED + prot := syscall.PROT_READ + switch { + case inprot© != 0: + prot |= syscall.PROT_WRITE + flags = syscall.MAP_PRIVATE + case inprot&RDWR != 0: + prot |= syscall.PROT_WRITE + } + if inprot&EXEC != 0 { + prot |= syscall.PROT_EXEC + } + if inflags&ANON != 0 { + flags |= syscall.MAP_ANON + } + + b, err := syscall.Mmap(int(fd), off, len, prot, flags) + if err != nil { + return nil, err + } + return b, nil +} + +func flush(addr, len uintptr) error { + _, _, errno := syscall.Syscall(_SYS_MSYNC, addr, len, _MS_SYNC) + if errno != 0 { + return syscall.Errno(errno) + } + return nil +} + +func lock(addr, len uintptr) error { + _, _, errno := syscall.Syscall(syscall.SYS_MLOCK, addr, len, 0) + if errno != 0 { + return syscall.Errno(errno) + } + return nil +} + +func unlock(addr, len uintptr) error { + _, _, errno := syscall.Syscall(syscall.SYS_MUNLOCK, addr, len, 0) + if errno != 0 { + return syscall.Errno(errno) + } + return nil +} + +func unmap(addr, len uintptr) error { + _, _, errno := syscall.Syscall(syscall.SYS_MUNMAP, addr, len, 0) + if errno != 0 { + return syscall.Errno(errno) + } + return nil +} diff --git a/vendor/github.com/edsrzf/mmap-go/mmap_windows.go b/vendor/github.com/edsrzf/mmap-go/mmap_windows.go new file mode 100644 index 00000000..c3d2d02d --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/mmap_windows.go @@ -0,0 +1,125 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mmap + +import ( + "errors" + "os" + "sync" + "syscall" +) + +// mmap on Windows is a two-step process. +// First, we call CreateFileMapping to get a handle. +// Then, we call MapviewToFile to get an actual pointer into memory. +// Because we want to emulate a POSIX-style mmap, we don't want to expose +// the handle -- only the pointer. We also want to return only a byte slice, +// not a struct, so it's convenient to manipulate. + +// We keep this map so that we can get back the original handle from the memory address. +var handleLock sync.Mutex +var handleMap = map[uintptr]syscall.Handle{} + +func mmap(len int, prot, flags, hfile uintptr, off int64) ([]byte, error) { + flProtect := uint32(syscall.PAGE_READONLY) + dwDesiredAccess := uint32(syscall.FILE_MAP_READ) + switch { + case prot© != 0: + flProtect = syscall.PAGE_WRITECOPY + dwDesiredAccess = syscall.FILE_MAP_COPY + case prot&RDWR != 0: + flProtect = syscall.PAGE_READWRITE + dwDesiredAccess = syscall.FILE_MAP_WRITE + } + if prot&EXEC != 0 { + flProtect <<= 4 + dwDesiredAccess |= syscall.FILE_MAP_EXECUTE + } + + // The maximum size is the area of the file, starting from 0, + // that we wish to allow to be mappable. It is the sum of + // the length the user requested, plus the offset where that length + // is starting from. This does not map the data into memory. + maxSizeHigh := uint32((off + int64(len)) >> 32) + maxSizeLow := uint32((off + int64(len)) & 0xFFFFFFFF) + // TODO: Do we need to set some security attributes? It might help portability. + h, errno := syscall.CreateFileMapping(syscall.Handle(hfile), nil, flProtect, maxSizeHigh, maxSizeLow, nil) + if h == 0 { + return nil, os.NewSyscallError("CreateFileMapping", errno) + } + + // Actually map a view of the data into memory. The view's size + // is the length the user requested. + fileOffsetHigh := uint32(off >> 32) + fileOffsetLow := uint32(off & 0xFFFFFFFF) + addr, errno := syscall.MapViewOfFile(h, dwDesiredAccess, fileOffsetHigh, fileOffsetLow, uintptr(len)) + if addr == 0 { + return nil, os.NewSyscallError("MapViewOfFile", errno) + } + handleLock.Lock() + handleMap[addr] = h + handleLock.Unlock() + + m := MMap{} + dh := m.header() + dh.Data = addr + dh.Len = len + dh.Cap = dh.Len + + return m, nil +} + +func flush(addr, len uintptr) error { + errno := syscall.FlushViewOfFile(addr, len) + if errno != nil { + return os.NewSyscallError("FlushViewOfFile", errno) + } + + handleLock.Lock() + defer handleLock.Unlock() + handle, ok := handleMap[addr] + if !ok { + // should be impossible; we would've errored above + return errors.New("unknown base address") + } + + errno = syscall.FlushFileBuffers(handle) + return os.NewSyscallError("FlushFileBuffers", errno) +} + +func lock(addr, len uintptr) error { + errno := syscall.VirtualLock(addr, len) + return os.NewSyscallError("VirtualLock", errno) +} + +func unlock(addr, len uintptr) error { + errno := syscall.VirtualUnlock(addr, len) + return os.NewSyscallError("VirtualUnlock", errno) +} + +func unmap(addr, len uintptr) error { + flush(addr, len) + // Lock the UnmapViewOfFile along with the handleMap deletion. + // As soon as we unmap the view, the OS is free to give the + // same addr to another new map. We don't want another goroutine + // to insert and remove the same addr into handleMap while + // we're trying to remove our old addr/handle pair. + handleLock.Lock() + defer handleLock.Unlock() + err := syscall.UnmapViewOfFile(addr) + if err != nil { + return err + } + + handle, ok := handleMap[addr] + if !ok { + // should be impossible; we would've errored above + return errors.New("unknown base address") + } + delete(handleMap, addr) + + e := syscall.CloseHandle(syscall.Handle(handle)) + return os.NewSyscallError("CloseHandle", e) +} diff --git a/vendor/github.com/edsrzf/mmap-go/msync_netbsd.go b/vendor/github.com/edsrzf/mmap-go/msync_netbsd.go new file mode 100644 index 00000000..a64b003e --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/msync_netbsd.go @@ -0,0 +1,8 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mmap + +const _SYS_MSYNC = 277 +const _MS_SYNC = 0x04 diff --git a/vendor/github.com/edsrzf/mmap-go/msync_unix.go b/vendor/github.com/edsrzf/mmap-go/msync_unix.go new file mode 100644 index 00000000..91ee5f40 --- /dev/null +++ b/vendor/github.com/edsrzf/mmap-go/msync_unix.go @@ -0,0 +1,14 @@ +// Copyright 2011 Evan Shaw. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux openbsd solaris + +package mmap + +import ( + "syscall" +) + +const _SYS_MSYNC = syscall.SYS_MSYNC +const _MS_SYNC = syscall.MS_SYNC diff --git a/vendor/github.com/fsnotify/fsnotify/AUTHORS b/vendor/github.com/fsnotify/fsnotify/AUTHORS index 0a5bf8f6..5ab5d41c 100644 --- a/vendor/github.com/fsnotify/fsnotify/AUTHORS +++ b/vendor/github.com/fsnotify/fsnotify/AUTHORS @@ -8,8 +8,10 @@ # Please keep the list sorted. +Aaron L Adrien Bustany Amit Krishnan +Anmol Sethi Bjørn Erik Pedersen Bruno Bigras Caleb Spare @@ -26,6 +28,7 @@ Kelvin Fo Ken-ichirou MATSUZAWA Matt Layher Nathan Youngman +Nickolai Zeldovich Patrick Paul Hammond Pawel Knap @@ -33,12 +36,15 @@ Pieter Droogendijk Pursuit92 Riku Voipio Rob Figueiredo +Rodrigo Chiossi Slawek Ligus Soge Zhang Tiffany Jernigan Tilak Sharma +Tom Payne Travis Cline Tudor Golubenco +Vahe Khachikyan Yukang bronze1man debrando diff --git a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md index 8c732c1d..be4d7ea2 100644 --- a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md +++ b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## v1.4.7 / 2018-01-09 + +* BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine) +* Tests: Fix missing verb on format string (thanks @rchiossi) +* Linux: Fix deadlock in Remove (thanks @aarondl) +* Linux: Watch.Add improvements (avoid race, fix consistency, reduce garbage) (thanks @twpayne) +* Docs: Moved FAQ into the README (thanks @vahe) +* Linux: Properly handle inotify's IN_Q_OVERFLOW event (thanks @zeldovich) +* Docs: replace references to OS X with macOS + ## v1.4.2 / 2016-10-10 * Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack) diff --git a/vendor/github.com/fsnotify/fsnotify/kqueue.go b/vendor/github.com/fsnotify/fsnotify/kqueue.go index c2b4acb1..86e76a3d 100644 --- a/vendor/github.com/fsnotify/fsnotify/kqueue.go +++ b/vendor/github.com/fsnotify/fsnotify/kqueue.go @@ -22,7 +22,7 @@ import ( type Watcher struct { Events chan Event Errors chan error - done chan bool // Channel for sending a "quit message" to the reader goroutine + done chan struct{} // Channel for sending a "quit message" to the reader goroutine kq int // File descriptor (as returned by the kqueue() syscall). @@ -56,7 +56,7 @@ func NewWatcher() (*Watcher, error) { externalWatches: make(map[string]bool), Events: make(chan Event), Errors: make(chan error), - done: make(chan bool), + done: make(chan struct{}), } go w.readEvents() @@ -71,10 +71,8 @@ func (w *Watcher) Close() error { return nil } w.isClosed = true - w.mu.Unlock() // copy paths to remove while locked - w.mu.Lock() var pathsToRemove = make([]string, 0, len(w.watches)) for name := range w.watches { pathsToRemove = append(pathsToRemove, name) @@ -82,15 +80,12 @@ func (w *Watcher) Close() error { w.mu.Unlock() // unlock before calling Remove, which also locks - var err error for _, name := range pathsToRemove { - if e := w.Remove(name); e != nil && err == nil { - err = e - } + w.Remove(name) } - // Send "quit" message to the reader goroutine: - w.done <- true + // send a "quit" message to the reader goroutine + close(w.done) return nil } @@ -266,17 +261,12 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { func (w *Watcher) readEvents() { eventBuffer := make([]unix.Kevent_t, 10) +loop: for { // See if there is a message on the "done" channel select { case <-w.done: - err := unix.Close(w.kq) - if err != nil { - w.Errors <- err - } - close(w.Events) - close(w.Errors) - return + break loop default: } @@ -284,7 +274,11 @@ func (w *Watcher) readEvents() { kevents, err := read(w.kq, eventBuffer, &keventWaitTime) // EINTR is okay, the syscall was interrupted before timeout expired. if err != nil && err != unix.EINTR { - w.Errors <- err + select { + case w.Errors <- err: + case <-w.done: + break loop + } continue } @@ -319,8 +313,12 @@ func (w *Watcher) readEvents() { if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { w.sendDirectoryChangeEvents(event.Name) } else { - // Send the event on the Events channel - w.Events <- event + // Send the event on the Events channel. + select { + case w.Events <- event: + case <-w.done: + break loop + } } if event.Op&Remove == Remove { @@ -352,6 +350,18 @@ func (w *Watcher) readEvents() { kevents = kevents[1:] } } + + // cleanup + err := unix.Close(w.kq) + if err != nil { + // only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors. + select { + case w.Errors <- err: + default: + } + } + close(w.Events) + close(w.Errors) } // newEvent returns an platform-independent Event based on kqueue Fflags. @@ -407,7 +417,11 @@ func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { // Get all files files, err := ioutil.ReadDir(dirPath) if err != nil { - w.Errors <- err + select { + case w.Errors <- err: + case <-w.done: + return + } } // Search for new files @@ -428,7 +442,11 @@ func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInf w.mu.Unlock() if !doesExist { // Send create event - w.Events <- newCreateEvent(filePath) + select { + case w.Events <- newCreateEvent(filePath): + case <-w.done: + return + } } // like watchDirectoryFiles (but without doing another ReadDir) diff --git a/vendor/github.com/glycerine/go-unsnap-stream/LICENSE b/vendor/github.com/glycerine/go-unsnap-stream/LICENSE new file mode 100644 index 00000000..31671ea6 --- /dev/null +++ b/vendor/github.com/glycerine/go-unsnap-stream/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2014 the go-unsnap-stream authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Permission is explicitly granted to relicense this material under new terms of +your choice when integrating this library with another library or project. diff --git a/vendor/github.com/glycerine/go-unsnap-stream/README.md b/vendor/github.com/glycerine/go-unsnap-stream/README.md new file mode 100644 index 00000000..b1b8c747 --- /dev/null +++ b/vendor/github.com/glycerine/go-unsnap-stream/README.md @@ -0,0 +1,20 @@ +go-unsnap-stream +================ + +This is a small golang library for decoding and encoding the snappy *streaming* format, specified here: https://github.com/google/snappy/blob/master/framing_format.txt + +Note that the *streaming or framing format* for snappy is different from snappy itself. Think of it as a train of boxcars: the streaming format breaks your data in chunks, applies snappy to each chunk alone, then puts a thin wrapper around the chunk, and sends it along in turn. You can begin decoding before receiving everything. And memory requirements for decoding are sane. + +Strangely, though the streaming format was first proposed in Go[1][2], it was never upated, and I could not locate any other library for Go that would handle the streaming/framed snappy format. Hence this implementation of the spec. There is a command line tool[3] that has a C implementation, but this is the only Go implementation that I am aware of. The reference for the framing/streaming spec seems to be the python implementation[4]. + +For binary compatibility with the python implementation, one could use the C-snappy compressor/decompressor code directly; using github.com/dgryski/go-csnappy. In fact we did this for a while to verify byte-for-byte compatiblity, as the native Go implementation produces slightly different binary compression (still conformant with the standard of course), which made test-diffs harder, and some have complained about it being slower than the C. + +However, while the c-snappy was useful for checking compatibility, it introduced dependencies on external C libraries (both the c-snappy library and the C standard library). Our go binary executable that used the go-unsnap-stream library was no longer standalone, and deployment was painful if not impossible if the target had a different C standard library. So we've gone back to using the snappy-go implementation (entirely in Go) for ease of deployment. See the comments at the top of unsnap.go if you wish to use c-snappy instead. + +[1] https://groups.google.com/forum/#!msg/snappy-compression/qvLNe2cSH9s/R19oBC-p7g4J + +[2] https://codereview.appspot.com/5167058 + +[3] https://github.com/kubo/snzip + +[4] https://pypi.python.org/pypi/python-snappy \ No newline at end of file diff --git a/vendor/github.com/glycerine/go-unsnap-stream/binary.dat b/vendor/github.com/glycerine/go-unsnap-stream/binary.dat new file mode 100644 index 00000000..f31eee2e Binary files /dev/null and b/vendor/github.com/glycerine/go-unsnap-stream/binary.dat differ diff --git a/vendor/github.com/glycerine/go-unsnap-stream/binary.dat.snappy b/vendor/github.com/glycerine/go-unsnap-stream/binary.dat.snappy new file mode 100644 index 00000000..ed370242 Binary files /dev/null and b/vendor/github.com/glycerine/go-unsnap-stream/binary.dat.snappy differ diff --git a/vendor/github.com/glycerine/go-unsnap-stream/rbuf.go b/vendor/github.com/glycerine/go-unsnap-stream/rbuf.go new file mode 100644 index 00000000..f771c392 --- /dev/null +++ b/vendor/github.com/glycerine/go-unsnap-stream/rbuf.go @@ -0,0 +1,375 @@ +package unsnap + +// copyright (c) 2014, Jason E. Aten +// license: MIT + +// Some text from the Golang standard library doc is adapted and +// reproduced in fragments below to document the expected behaviors +// of the interface functions Read()/Write()/ReadFrom()/WriteTo() that +// are implemented here. Those descriptions (see +// http://golang.org/pkg/io/#Reader for example) are +// copyright 2010 The Go Authors. + +import "io" + +// FixedSizeRingBuf: +// +// a fixed-size circular ring buffer. Yes, just what is says. +// +// We keep a pair of ping/pong buffers so that we can linearize +// the circular buffer into a contiguous slice if need be. +// +// For efficiency, a FixedSizeRingBuf may be vastly preferred to +// a bytes.Buffer. The ReadWithoutAdvance(), Advance(), and Adopt() +// methods are all non-standard methods written for speed. +// +// For an I/O heavy application, I have replaced bytes.Buffer with +// FixedSizeRingBuf and seen memory consumption go from 8GB to 25MB. +// Yes, that is a 300x reduction in memory footprint. Everything ran +// faster too. +// +// Note that Bytes(), while inescapable at times, is expensive: avoid +// it if possible. Instead it is better to use the FixedSizeRingBuf.Readable +// member to get the number of bytes available. Bytes() is expensive because +// it may copy the back and then the front of a wrapped buffer A[Use] +// into A[1-Use] in order to get a contiguous slice. If possible use ContigLen() +// first to get the size that can be read without copying, Read() that +// amount, and then Read() a second time -- to avoid the copy. + +type FixedSizeRingBuf struct { + A [2][]byte // a pair of ping/pong buffers. Only one is active. + Use int // which A buffer is in active use, 0 or 1 + N int // MaxViewInBytes, the size of A[0] and A[1] in bytes. + Beg int // start of data in A[Use] + Readable int // number of bytes available to read in A[Use] + + OneMade bool // lazily instantiate the [1] buffer. If we never call Bytes(), + // we may never need it. If OneMade is false, the Use must be = 0. +} + +func (b *FixedSizeRingBuf) Make2ndBuffer() { + if b.OneMade { + return + } + b.A[1] = make([]byte, b.N, b.N) + b.OneMade = true +} + +// get the length of the largest read that we can provide to a contiguous slice +// without an extra linearizing copy of all bytes internally. +func (b *FixedSizeRingBuf) ContigLen() int { + extent := b.Beg + b.Readable + firstContigLen := intMin(extent, b.N) - b.Beg + return firstContigLen +} + +func NewFixedSizeRingBuf(maxViewInBytes int) *FixedSizeRingBuf { + n := maxViewInBytes + r := &FixedSizeRingBuf{ + Use: 0, // 0 or 1, whichever is actually in use at the moment. + // If we are asked for Bytes() and we wrap, linearize into the other. + + N: n, + Beg: 0, + Readable: 0, + OneMade: false, + } + r.A[0] = make([]byte, n, n) + + // r.A[1] initialized lazily now. + + return r +} + +// from the standard library description of Bytes(): +// Bytes() returns a slice of the contents of the unread portion of the buffer. +// If the caller changes the contents of the +// returned slice, the contents of the buffer will change provided there +// are no intervening method calls on the Buffer. +// +func (b *FixedSizeRingBuf) Bytes() []byte { + + extent := b.Beg + b.Readable + if extent <= b.N { + // we fit contiguously in this buffer without wrapping to the other + return b.A[b.Use][b.Beg:(b.Beg + b.Readable)] + } + + // wrap into the other buffer + b.Make2ndBuffer() + + src := b.Use + dest := 1 - b.Use + + n := copy(b.A[dest], b.A[src][b.Beg:]) + n += copy(b.A[dest][n:], b.A[src][0:(extent%b.N)]) + + b.Use = dest + b.Beg = 0 + + return b.A[b.Use][:n] +} + +// Read(): +// +// from bytes.Buffer.Read(): Read reads the next len(p) bytes +// from the buffer or until the buffer is drained. The return +// value n is the number of bytes read. If the buffer has no data +// to return, err is io.EOF (unless len(p) is zero); otherwise it is nil. +// +// from the description of the Reader interface, +// http://golang.org/pkg/io/#Reader +// +/* +Reader is the interface that wraps the basic Read method. + +Read reads up to len(p) bytes into p. It returns the number +of bytes read (0 <= n <= len(p)) and any error encountered. +Even if Read returns n < len(p), it may use all of p as scratch +space during the call. If some data is available but not +len(p) bytes, Read conventionally returns what is available +instead of waiting for more. + +When Read encounters an error or end-of-file condition after +successfully reading n > 0 bytes, it returns the number of bytes +read. It may return the (non-nil) error from the same call or +return the error (and n == 0) from a subsequent call. An instance +of this general case is that a Reader returning a non-zero number +of bytes at the end of the input stream may return +either err == EOF or err == nil. The next Read should +return 0, EOF regardless. + +Callers should always process the n > 0 bytes returned before +considering the error err. Doing so correctly handles I/O errors +that happen after reading some bytes and also both of the +allowed EOF behaviors. + +Implementations of Read are discouraged from returning a zero +byte count with a nil error, and callers should treat that +situation as a no-op. +*/ +// + +func (b *FixedSizeRingBuf) Read(p []byte) (n int, err error) { + return b.ReadAndMaybeAdvance(p, true) +} + +// if you want to Read the data and leave it in the buffer, so as +// to peek ahead for example. +func (b *FixedSizeRingBuf) ReadWithoutAdvance(p []byte) (n int, err error) { + return b.ReadAndMaybeAdvance(p, false) +} + +func (b *FixedSizeRingBuf) ReadAndMaybeAdvance(p []byte, doAdvance bool) (n int, err error) { + if len(p) == 0 { + return 0, nil + } + if b.Readable == 0 { + return 0, io.EOF + } + extent := b.Beg + b.Readable + if extent <= b.N { + n += copy(p, b.A[b.Use][b.Beg:extent]) + } else { + n += copy(p, b.A[b.Use][b.Beg:b.N]) + if n < len(p) { + n += copy(p[n:], b.A[b.Use][0:(extent%b.N)]) + } + } + if doAdvance { + b.Advance(n) + } + return +} + +// +// Write writes len(p) bytes from p to the underlying data stream. +// It returns the number of bytes written from p (0 <= n <= len(p)) +// and any error encountered that caused the write to stop early. +// Write must return a non-nil error if it returns n < len(p). +// +func (b *FixedSizeRingBuf) Write(p []byte) (n int, err error) { + for { + if len(p) == 0 { + // nothing (left) to copy in; notice we shorten our + // local copy p (below) as we read from it. + return + } + + writeCapacity := b.N - b.Readable + if writeCapacity <= 0 { + // we are all full up already. + return n, io.ErrShortWrite + } + if len(p) > writeCapacity { + err = io.ErrShortWrite + // leave err set and + // keep going, write what we can. + } + + writeStart := (b.Beg + b.Readable) % b.N + + upperLim := intMin(writeStart+writeCapacity, b.N) + + k := copy(b.A[b.Use][writeStart:upperLim], p) + + n += k + b.Readable += k + p = p[k:] + + // we can fill from b.A[b.Use][0:something] from + // p's remainder, so loop + } +} + +// WriteTo and ReadFrom avoid intermediate allocation and copies. + +// WriteTo writes data to w until there's no more data to write +// or when an error occurs. The return value n is the number of +// bytes written. Any error encountered during the write is also returned. +func (b *FixedSizeRingBuf) WriteTo(w io.Writer) (n int64, err error) { + + if b.Readable == 0 { + return 0, io.EOF + } + + extent := b.Beg + b.Readable + firstWriteLen := intMin(extent, b.N) - b.Beg + secondWriteLen := b.Readable - firstWriteLen + if firstWriteLen > 0 { + m, e := w.Write(b.A[b.Use][b.Beg:(b.Beg + firstWriteLen)]) + n += int64(m) + b.Advance(m) + + if e != nil { + return n, e + } + // all bytes should have been written, by definition of + // Write method in io.Writer + if m != firstWriteLen { + return n, io.ErrShortWrite + } + } + if secondWriteLen > 0 { + m, e := w.Write(b.A[b.Use][0:secondWriteLen]) + n += int64(m) + b.Advance(m) + + if e != nil { + return n, e + } + // all bytes should have been written, by definition of + // Write method in io.Writer + if m != secondWriteLen { + return n, io.ErrShortWrite + } + } + + return n, nil +} + +// ReadFrom() reads data from r until EOF or error. The return value n +// is the number of bytes read. Any error except io.EOF encountered +// during the read is also returned. +func (b *FixedSizeRingBuf) ReadFrom(r io.Reader) (n int64, err error) { + for { + writeCapacity := b.N - b.Readable + if writeCapacity <= 0 { + // we are all full + return n, nil + } + writeStart := (b.Beg + b.Readable) % b.N + upperLim := intMin(writeStart+writeCapacity, b.N) + + m, e := r.Read(b.A[b.Use][writeStart:upperLim]) + n += int64(m) + b.Readable += m + if e == io.EOF { + return n, nil + } + if e != nil { + return n, e + } + } +} + +func (b *FixedSizeRingBuf) Reset() { + b.Beg = 0 + b.Readable = 0 + b.Use = 0 +} + +// Advance(): non-standard, but better than Next(), +// because we don't have to unwrap our buffer and pay the cpu time +// for the copy that unwrapping may need. +// Useful in conjuction/after ReadWithoutAdvance() above. +func (b *FixedSizeRingBuf) Advance(n int) { + if n <= 0 { + return + } + if n > b.Readable { + n = b.Readable + } + b.Readable -= n + b.Beg = (b.Beg + n) % b.N +} + +// Adopt(): non-standard. +// +// For efficiency's sake, (possibly) take ownership of +// already allocated slice offered in me. +// +// If me is large we will adopt it, and we will potentially then +// write to the me buffer. +// If we already have a bigger buffer, copy me into the existing +// buffer instead. +func (b *FixedSizeRingBuf) Adopt(me []byte) { + n := len(me) + if n > b.N { + b.A[0] = me + b.OneMade = false + b.N = n + b.Use = 0 + b.Beg = 0 + b.Readable = n + } else { + // we already have a larger buffer, reuse it. + copy(b.A[0], me) + b.Use = 0 + b.Beg = 0 + b.Readable = n + } +} + +func intMax(a, b int) int { + if a > b { + return a + } else { + return b + } +} + +func intMin(a, b int) int { + if a < b { + return a + } else { + return b + } +} + +// Get the (beg, end] indices of the tailing empty buffer of bytes slice that from that is free for writing. +// Note: not guaranteed to be zeroed. At all. +func (b *FixedSizeRingBuf) GetEndmostWritable() (beg int, end int) { + extent := b.Beg + b.Readable + if extent < b.N { + return extent, b.N + } + + return extent % b.N, b.Beg +} + +// Note: not guaranteed to be zeroed. +func (b *FixedSizeRingBuf) GetEndmostWritableSlice() []byte { + beg, e := b.GetEndmostWritable() + return b.A[b.Use][beg:e] +} diff --git a/vendor/github.com/glycerine/go-unsnap-stream/snap.go b/vendor/github.com/glycerine/go-unsnap-stream/snap.go new file mode 100644 index 00000000..12a8d40b --- /dev/null +++ b/vendor/github.com/glycerine/go-unsnap-stream/snap.go @@ -0,0 +1,100 @@ +package unsnap + +import ( + "encoding/binary" + + // no c lib dependency + snappy "github.com/golang/snappy" + // or, use the C wrapper for speed + //snappy "github.com/dgryski/go-csnappy" +) + +// add Write() method for SnappyFile (see unsnap.go) + +// reference for snappy framing/streaming format: +// http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +// ?spec=svn68&r=71 + +// +// Write writes len(p) bytes from p to the underlying data stream. +// It returns the number of bytes written from p (0 <= n <= len(p)) and +// any error encountered that caused the write to stop early. Write +// must return a non-nil error if it returns n < len(p). +// +func (sf *SnappyFile) Write(p []byte) (n int, err error) { + + if sf.SnappyEncodeDecodeOff { + return sf.Writer.Write(p) + } + + if !sf.Writing { + panic("Writing on a read-only SnappyFile") + } + + // encoding in snappy can apparently go beyond the original size, beware. + // so our buffers must be sized 2*max snappy chunk => 2 * CHUNK_MAX(65536) + + sf.DecBuf.Reset() + sf.EncBuf.Reset() + + if !sf.HeaderChunkWritten { + sf.HeaderChunkWritten = true + _, err = sf.Writer.Write(SnappyStreamHeaderMagic) + if err != nil { + return + } + } + var chunk []byte + var chunk_type byte + var crc uint32 + + for len(p) > 0 { + + // chunk points to input p by default, unencoded input. + chunk = p[:IntMin(len(p), CHUNK_MAX)] + crc = masked_crc32c(chunk) + + writeme := chunk[:] + + // first write to EncBuf, as a temp, in case we want + // to discard and send uncompressed instead. + compressed_chunk := snappy.Encode(sf.EncBuf.GetEndmostWritableSlice(), chunk) + + if len(compressed_chunk) <= int((1-_COMPRESSION_THRESHOLD)*float64(len(chunk))) { + writeme = compressed_chunk + chunk_type = _COMPRESSED_CHUNK + } else { + // keep writeme pointing at original chunk (uncompressed) + chunk_type = _UNCOMPRESSED_CHUNK + } + + const crc32Sz = 4 + var tag32 uint32 = uint32(chunk_type) + (uint32(len(writeme)+crc32Sz) << 8) + + err = binary.Write(sf.Writer, binary.LittleEndian, tag32) + if err != nil { + return + } + + err = binary.Write(sf.Writer, binary.LittleEndian, crc) + if err != nil { + return + } + + _, err = sf.Writer.Write(writeme) + if err != nil { + return + } + + n += len(chunk) + p = p[len(chunk):] + } + return n, nil +} + +func IntMin(a int, b int) int { + if a < b { + return a + } + return b +} diff --git a/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt b/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt new file mode 100644 index 00000000..5f502793 --- /dev/null +++ b/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt @@ -0,0 +1 @@ +hello_snappy diff --git a/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt.snappy b/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt.snappy new file mode 100644 index 00000000..ba45ecd4 Binary files /dev/null and b/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt.snappy differ diff --git a/vendor/github.com/glycerine/go-unsnap-stream/unsnap.go b/vendor/github.com/glycerine/go-unsnap-stream/unsnap.go new file mode 100644 index 00000000..8789445c --- /dev/null +++ b/vendor/github.com/glycerine/go-unsnap-stream/unsnap.go @@ -0,0 +1,513 @@ +package unsnap + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "os" + + "hash/crc32" + + snappy "github.com/golang/snappy" + // The C library can be used, but this makes the binary dependent + // lots of extraneous c-libraries; it is no longer stand-alone. Yuck. + // + // Therefore we comment out the "dgryski/go-csnappy" path and use the + // "github.com/golang/snappy/snappy" above instead. If you are + // performance limited and can deal with distributing more libraries, + // then this is easy to swap. + // + // If you swap, note that some of the tests won't pass + // because snappy-go produces slightly different (but still + // conformant) encodings on some data. Here are bindings + // to the C-snappy: + // snappy "github.com/dgryski/go-csnappy" +) + +// SnappyFile: create a drop-in-replacement/wrapper for an *os.File that handles doing the unsnappification online as more is read from it + +type SnappyFile struct { + Fname string + + Reader io.Reader + Writer io.Writer + + // allow clients to substitute us for an os.File and just switch + // off compression if they don't want it. + SnappyEncodeDecodeOff bool // if true, we bypass straight to Filep + + EncBuf FixedSizeRingBuf // holds any extra that isn't yet returned, encoded + DecBuf FixedSizeRingBuf // holds any extra that isn't yet returned, decoded + + // for writing to stream-framed snappy + HeaderChunkWritten bool + + // Sanity check: we can only read, or only write, to one SnappyFile. + // EncBuf and DecBuf are used differently in each mode. Verify + // that we are consistent with this flag. + Writing bool +} + +var total int + +// for debugging, show state of buffers +func (f *SnappyFile) Dump() { + fmt.Printf("EncBuf has length %d and contents:\n%s\n", len(f.EncBuf.Bytes()), string(f.EncBuf.Bytes())) + fmt.Printf("DecBuf has length %d and contents:\n%s\n", len(f.DecBuf.Bytes()), string(f.DecBuf.Bytes())) +} + +func (f *SnappyFile) Read(p []byte) (n int, err error) { + + if f.SnappyEncodeDecodeOff { + return f.Reader.Read(p) + } + + if f.Writing { + panic("Reading on a write-only SnappyFile") + } + + // before we unencrypt more, try to drain the DecBuf first + n, _ = f.DecBuf.Read(p) + if n > 0 { + total += n + return n, nil + } + + //nEncRead, nDecAdded, err := UnsnapOneFrame(f.Filep, &f.EncBuf, &f.DecBuf, f.Fname) + _, _, err = UnsnapOneFrame(f.Reader, &f.EncBuf, &f.DecBuf, f.Fname) + if err != nil && err != io.EOF { + panic(err) + } + + n, _ = f.DecBuf.Read(p) + + if n > 0 { + total += n + return n, nil + } + if f.DecBuf.Readable == 0 { + if f.DecBuf.Readable == 0 && f.EncBuf.Readable == 0 { + // only now (when EncBuf is empty) can we give io.EOF. + // Any earlier, and we leave stuff un-decoded! + return 0, io.EOF + } + } + return 0, nil +} + +func Open(name string) (file *SnappyFile, err error) { + fp, err := os.Open(name) + if err != nil { + return nil, err + } + // encoding in snappy can apparently go beyond the original size, so + // we make our buffers big enough, 2*max snappy chunk => 2 * CHUNK_MAX(65536) + + snap := NewReader(fp) + snap.Fname = name + return snap, nil +} + +func NewReader(r io.Reader) *SnappyFile { + return &SnappyFile{ + Reader: r, + EncBuf: *NewFixedSizeRingBuf(CHUNK_MAX * 2), // buffer of snappy encoded bytes + DecBuf: *NewFixedSizeRingBuf(CHUNK_MAX * 2), // buffer of snapppy decoded bytes + Writing: false, + } +} + +func NewWriter(w io.Writer) *SnappyFile { + return &SnappyFile{ + Writer: w, + EncBuf: *NewFixedSizeRingBuf(65536), // on writing: temp for testing compression + DecBuf: *NewFixedSizeRingBuf(65536 * 2), // on writing: final buffer of snappy framed and encoded bytes + Writing: true, + } +} + +func Create(name string) (file *SnappyFile, err error) { + fp, err := os.Create(name) + if err != nil { + return nil, err + } + snap := NewWriter(fp) + snap.Fname = name + return snap, nil +} + +func (f *SnappyFile) Close() error { + if f.Writing { + wc, ok := f.Writer.(io.WriteCloser) + if ok { + return wc.Close() + } + return nil + } + rc, ok := f.Reader.(io.ReadCloser) + if ok { + return rc.Close() + } + return nil +} + +func (f *SnappyFile) Sync() error { + file, ok := f.Writer.(*os.File) + if ok { + return file.Sync() + } + return nil +} + +// for an increment of a frame at a time: +// read from r into encBuf (encBuf is still encoded, thus the name), and write unsnappified frames into outDecodedBuf +// the returned n: number of bytes read from the encrypted encBuf +func UnsnapOneFrame(r io.Reader, encBuf *FixedSizeRingBuf, outDecodedBuf *FixedSizeRingBuf, fname string) (nEnc int64, nDec int64, err error) { + // b, err := ioutil.ReadAll(r) + // if err != nil { + // panic(err) + // } + + nEnc = 0 + nDec = 0 + + // read up to 65536 bytes from r into encBuf, at least a snappy frame + nread, err := io.CopyN(encBuf, r, 65536) // returns nwrotebytes, err + nEnc += nread + if err != nil { + if err == io.EOF { + if nread == 0 { + if encBuf.Readable == 0 { + return nEnc, nDec, io.EOF + } + // else we have bytes in encBuf, so decode them! + err = nil + } else { + // continue below, processing the nread bytes + err = nil + } + } else { + panic(err) + } + } + + // flag for printing chunk size alignment messages + verbose := false + + const snappyStreamHeaderSz = 10 + const headerSz = 4 + const crc32Sz = 4 + // the magic 18 bytes accounts for the snappy streaming header and the first chunks size and checksum + // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt + + chunk := (*encBuf).Bytes() + + // however we exit, advance as + // defer func() { (*encBuf).Next(N) }() + + // 65536 is the max size of a snappy framed chunk. See + // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt:91 + // buf := make([]byte, 65536) + + // fmt.Printf("read from file, b is len:%d with value: %#v\n", len(b), b) + // fmt.Printf("read from file, bcut is len:%d with value: %#v\n", len(bcut), bcut) + + //fmt.Printf("raw bytes of chunksz are: %v\n", b[11:14]) + + fourbytes := make([]byte, 4) + chunkCount := 0 + + for nDec < 65536 { + if len(chunk) == 0 { + break + } + chunkCount++ + fourbytes[3] = 0 + copy(fourbytes, chunk[1:4]) + chunksz := binary.LittleEndian.Uint32(fourbytes) + chunk_type := chunk[0] + + switch true { + case chunk_type == 0xff: + { // stream identifier + + streamHeader := chunk[:snappyStreamHeaderSz] + if 0 != bytes.Compare(streamHeader, []byte{0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59}) { + panic("file had chunk starting with 0xff but then no magic snappy streaming protocol bytes, aborting.") + } else { + //fmt.Printf("got streaming snappy magic header just fine.\n") + } + chunk = chunk[snappyStreamHeaderSz:] + (*encBuf).Advance(snappyStreamHeaderSz) + nEnc += snappyStreamHeaderSz + continue + } + case chunk_type == 0x00: + { // compressed data + if verbose { + fmt.Fprintf(os.Stderr, "chunksz is %d while total bytes avail are: %d\n", int(chunksz), len(chunk)-4) + } + + crc := binary.LittleEndian.Uint32(chunk[headerSz:(headerSz + crc32Sz)]) + section := chunk[(headerSz + crc32Sz):(headerSz + chunksz)] + + dec, ok := snappy.Decode(nil, section) + if ok != nil { + // we've probably truncated a snappy frame at this point + // ok=snappy: corrupt input + // len(dec) == 0 + // + panic(fmt.Sprintf("could not decode snappy stream: '%s' and len dec=%d and ok=%v\n", fname, len(dec), ok)) + + // get back to caller with what we've got so far + return nEnc, nDec, nil + } + // fmt.Printf("ok, b is %#v , %#v\n", ok, dec) + + // spit out decoded text + // n, err := w.Write(dec) + //fmt.Printf("len(dec) = %d, outDecodedBuf.Readable=%d\n", len(dec), outDecodedBuf.Readable) + bnb := bytes.NewBuffer(dec) + n, err := io.Copy(outDecodedBuf, bnb) + if err != nil { + //fmt.Printf("got n=%d, err= %s ; when trying to io.Copy(outDecodedBuf: N=%d, Readable=%d)\n", n, err, outDecodedBuf.N, outDecodedBuf.Readable) + panic(err) + } + if n != int64(len(dec)) { + panic("could not write all bytes to outDecodedBuf") + } + nDec += n + + // verify the crc32 rotated checksum + m32 := masked_crc32c(dec) + if m32 != crc { + panic(fmt.Sprintf("crc32 masked failiure. expected: %v but got: %v", crc, m32)) + } else { + //fmt.Printf("\nchecksums match: %v == %v\n", crc, m32) + } + + // move to next header + inc := (headerSz + int(chunksz)) + chunk = chunk[inc:] + (*encBuf).Advance(inc) + nEnc += int64(inc) + continue + } + case chunk_type == 0x01: + { // uncompressed data + + //n, err := w.Write(chunk[(headerSz+crc32Sz):(headerSz + int(chunksz))]) + n, err := io.Copy(outDecodedBuf, bytes.NewBuffer(chunk[(headerSz+crc32Sz):(headerSz+int(chunksz))])) + if verbose { + //fmt.Printf("debug: n=%d err=%v chunksz=%d outDecodedBuf='%v'\n", n, err, chunksz, outDecodedBuf) + } + if err != nil { + panic(err) + } + if n != int64(chunksz-crc32Sz) { + panic("could not write all bytes to stdout") + } + nDec += n + + inc := (headerSz + int(chunksz)) + chunk = chunk[inc:] + (*encBuf).Advance(inc) + nEnc += int64(inc) + continue + } + case chunk_type == 0xfe: + fallthrough // padding, just skip it + case chunk_type >= 0x80 && chunk_type <= 0xfd: + { // Reserved skippable chunks + //fmt.Printf("\nin reserved skippable chunks, at nEnc=%v\n", nEnc) + inc := (headerSz + int(chunksz)) + chunk = chunk[inc:] + nEnc += int64(inc) + (*encBuf).Advance(inc) + continue + } + + default: + panic(fmt.Sprintf("unrecognized/unsupported chunk type %#v", chunk_type)) + } + + } // end for{} + + return nEnc, nDec, err + //return int64(N), nil +} + +// for whole file at once: +// +// receive on stdin a stream of bytes in the snappy-streaming framed +// format, defined here: http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt +// Grab each frame, run it through the snappy decoder, and spit out +// each frame all joined back-to-back on stdout. +// +func Unsnappy(r io.Reader, w io.Writer) (err error) { + b, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + + // flag for printing chunk size alignment messages + verbose := false + + const snappyStreamHeaderSz = 10 + const headerSz = 4 + const crc32Sz = 4 + // the magic 18 bytes accounts for the snappy streaming header and the first chunks size and checksum + // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt + + chunk := b[:] + + // 65536 is the max size of a snappy framed chunk. See + // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt:91 + //buf := make([]byte, 65536) + + // fmt.Printf("read from file, b is len:%d with value: %#v\n", len(b), b) + // fmt.Printf("read from file, bcut is len:%d with value: %#v\n", len(bcut), bcut) + + //fmt.Printf("raw bytes of chunksz are: %v\n", b[11:14]) + + fourbytes := make([]byte, 4) + chunkCount := 0 + + for { + if len(chunk) == 0 { + break + } + chunkCount++ + fourbytes[3] = 0 + copy(fourbytes, chunk[1:4]) + chunksz := binary.LittleEndian.Uint32(fourbytes) + chunk_type := chunk[0] + + switch true { + case chunk_type == 0xff: + { // stream identifier + + streamHeader := chunk[:snappyStreamHeaderSz] + if 0 != bytes.Compare(streamHeader, []byte{0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59}) { + panic("file had chunk starting with 0xff but then no magic snappy streaming protocol bytes, aborting.") + } else { + //fmt.Printf("got streaming snappy magic header just fine.\n") + } + chunk = chunk[snappyStreamHeaderSz:] + continue + } + case chunk_type == 0x00: + { // compressed data + if verbose { + fmt.Fprintf(os.Stderr, "chunksz is %d while total bytes avail are: %d\n", int(chunksz), len(chunk)-4) + } + + //crc := binary.LittleEndian.Uint32(chunk[headerSz:(headerSz + crc32Sz)]) + section := chunk[(headerSz + crc32Sz):(headerSz + chunksz)] + + dec, ok := snappy.Decode(nil, section) + if ok != nil { + panic("could not decode snappy stream") + } + // fmt.Printf("ok, b is %#v , %#v\n", ok, dec) + + // spit out decoded text + n, err := w.Write(dec) + if err != nil { + panic(err) + } + if n != len(dec) { + panic("could not write all bytes to stdout") + } + + // TODO: verify the crc32 rotated checksum? + + // move to next header + chunk = chunk[(headerSz + int(chunksz)):] + continue + } + case chunk_type == 0x01: + { // uncompressed data + + //crc := binary.LittleEndian.Uint32(chunk[headerSz:(headerSz + crc32Sz)]) + section := chunk[(headerSz + crc32Sz):(headerSz + chunksz)] + + n, err := w.Write(section) + if err != nil { + panic(err) + } + if n != int(chunksz-crc32Sz) { + panic("could not write all bytes to stdout") + } + + chunk = chunk[(headerSz + int(chunksz)):] + continue + } + case chunk_type == 0xfe: + fallthrough // padding, just skip it + case chunk_type >= 0x80 && chunk_type <= 0xfd: + { // Reserved skippable chunks + chunk = chunk[(headerSz + int(chunksz)):] + continue + } + + default: + panic(fmt.Sprintf("unrecognized/unsupported chunk type %#v", chunk_type)) + } + + } // end for{} + + return nil +} + +// 0xff 0x06 0x00 0x00 sNaPpY +var SnappyStreamHeaderMagic = []byte{0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59} + +const CHUNK_MAX = 65536 +const _STREAM_TO_STREAM_BLOCK_SIZE = CHUNK_MAX +const _STREAM_IDENTIFIER = `sNaPpY` +const _COMPRESSED_CHUNK = 0x00 +const _UNCOMPRESSED_CHUNK = 0x01 +const _IDENTIFIER_CHUNK = 0xff +const _RESERVED_UNSKIPPABLE0 = 0x02 // chunk ranges are [inclusive, exclusive) +const _RESERVED_UNSKIPPABLE1 = 0x80 +const _RESERVED_SKIPPABLE0 = 0x80 +const _RESERVED_SKIPPABLE1 = 0xff + +// the minimum percent of bytes compression must save to be enabled in automatic +// mode +const _COMPRESSION_THRESHOLD = .125 + +var crctab *crc32.Table + +func init() { + crctab = crc32.MakeTable(crc32.Castagnoli) // this is correct table, matches the crc32c.c code used by python +} + +func masked_crc32c(data []byte) uint32 { + + // see the framing format specification, http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt + var crc uint32 = crc32.Checksum(data, crctab) + return (uint32((crc>>15)|(crc<<17)) + 0xa282ead8) +} + +func ReadSnappyStreamCompressedFile(filename string) ([]byte, error) { + + snappyFile, err := Open(filename) + if err != nil { + return []byte{}, err + } + + var bb bytes.Buffer + _, err = bb.ReadFrom(snappyFile) + if err == io.EOF { + err = nil + } + if err != nil { + panic(err) + } + + return bb.Bytes(), err +} diff --git a/vendor/github.com/golang-commonmark/markdown/LICENSE b/vendor/github.com/golang-commonmark/markdown/LICENSE deleted file mode 100644 index 9cdcd201..00000000 --- a/vendor/github.com/golang-commonmark/markdown/LICENSE +++ /dev/null @@ -1,10 +0,0 @@ -Copyright (c) 2015, The Authors -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/golang-commonmark/markdown/byteutil/byteutil.go b/vendor/github.com/golang-commonmark/markdown/byteutil/byteutil.go deleted file mode 100644 index 114f72bf..00000000 --- a/vendor/github.com/golang-commonmark/markdown/byteutil/byteutil.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2015 The Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package byteutil provides various operations on bytes and byte strings. -package byteutil - -var ( - digit [256]bool - hexdigit [256]bool - letter [256]bool - uppercase [256]bool - lowercase [256]bool - alphanum [256]bool - tolower [256]byte - toupper [256]byte -) - -func init() { - for _, b := range "0123456789" { - digit[b] = true - } - for _, b := range "0123456789abcdefABCDEF" { - hexdigit[b] = true - } - for _, b := range "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" { - letter[b] = true - } - for _, b := range "abcdefghijklmnopqrstuvwxyz" { - lowercase[b] = true - } - for _, b := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ" { - uppercase[b] = true - } - for _, b := range "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" { - alphanum[b] = true - } - for i := 0; i < 256; i++ { - tolower[i] = byte(i) - toupper[i] = byte(i) - } - for _, b := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ" { - tolower[b] = byte(b) - 'A' + 'a' - } - for _, b := range "abcdefghijklmnopqrstuvwxyz" { - toupper[b] = byte(b) - 'a' + 'A' - } -} - -func IsDigit(b byte) bool { - return digit[b] -} - -func IsHexDigit(b byte) bool { - return hexdigit[b] -} - -func IsLetter(b byte) bool { - return letter[b] -} - -func IsLowercaseLetter(b byte) bool { - return lowercase[b] -} - -func IsUppercaseLetter(b byte) bool { - return uppercase[b] -} - -func IsAlphaNum(b byte) bool { - return alphanum[b] -} - -func ToLower(s string) string { - if s == "" { - return "" - } - - hasUpper := false - for i := 0; i < len(s); i++ { - if uppercase[s[i]] { - hasUpper = true - break - } - } - if !hasUpper { - return s - } - - buf := make([]byte, len(s)) - for i := 0; i < len(s); i++ { - buf[i] = tolower[s[i]] - } - return string(buf) -} - -func ToUpper(s string) string { - if s == "" { - return "" - } - - hasLower := false - for i := 0; i < len(s); i++ { - if lowercase[s[i]] { - hasLower = true - break - } - } - if !hasLower { - return s - } - - buf := make([]byte, len(s)) - for i := 0; i < len(s); i++ { - buf[i] = toupper[s[i]] - } - return string(buf) -} - -func ByteToLower(b byte) byte { - return tolower[b] -} - -func ByteToUpper(b byte) byte { - return toupper[b] -} - -func IndexAny(s, chars string) int { - var t [256]bool - for i := 0; i < len(chars); i++ { - t[chars[i]] = true - } - for i := 0; i < len(s); i++ { - if t[s[i]] { - return i - } - } - return -1 -} - -func IndexAnyTable(s string, t *[256]bool) int { - for i := 0; i < len(s); i++ { - if t[s[i]] { - return i - } - } - return -1 -} - -func Unhex(d byte) byte { - switch { - case digit[d]: - return d - '0' - case uppercase[d]: - return d - 'A' + 10 - case lowercase[d]: - return d - 'a' + 10 - } - panic("unhex: not hex digit") -} diff --git a/vendor/github.com/golang-commonmark/markdown/linkify/charset.go b/vendor/github.com/golang-commonmark/markdown/linkify/charset.go deleted file mode 100644 index 82a7c915..00000000 --- a/vendor/github.com/golang-commonmark/markdown/linkify/charset.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2015 The Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package linkify - -import "unicode" - -var ( - trigger [256]bool - unreserved [256]bool - subdelims [256]bool - emailcs [256]bool - basicPunct [256]bool -) - -func init() { - for _, b := range "-._~" { - unreserved[b] = true - } - for _, b := range "!$&'()*+,;=" { - subdelims[b] = true - } - for _, b := range "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%&'*+/=?^_`{|}~-" { - emailcs[b] = true - } - for _, b := range ".,?!;:" { - basicPunct[b] = true - } -} - -func isAllowedInEmail(r rune) bool { - return r < 0x7f && emailcs[r] -} - -func isLetterOrDigit(r rune) bool { - return unicode.In(r, unicode.Letter, unicode.Digit) -} - -func isPunctOrSpaceOrControl(r rune) bool { - return r == '<' || r == '>' || unicode.In(r, unicode.Punct, unicode.Space, unicode.Cc) -} - -func isUnreserved(r rune) bool { - return (r < 0x7f && unreserved[r]) || isLetterOrDigit(r) -} - -func isSubDelimiter(r rune) bool { - return r < 0x7f && subdelims[r] -} diff --git a/vendor/github.com/golang-commonmark/markdown/linkify/email.go b/vendor/github.com/golang-commonmark/markdown/linkify/email.go deleted file mode 100644 index 35bfb679..00000000 --- a/vendor/github.com/golang-commonmark/markdown/linkify/email.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2015 The Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package linkify - -import ( - "unicode" - "unicode/utf8" -) - -func findEmailStart(s string, start int) (_ int, _ bool) { - end := start - allowDot := false - for end >= 0 { - b := s[end] - switch { - case emailcs[b]: - allowDot = true - case b == '.': - if !allowDot { - return - } - allowDot = false - default: - if end == start { - return - } - if s[end+1] == '.' { - return - } - r, _ := utf8.DecodeLastRuneInString(s[:end+1]) - if r == utf8.RuneError { - return - } - if !unicode.IsSpace(r) { - return - } - return end + 1, true - } - end-- - } - if end < start && s[end+1] == '.' { - return - } - return end + 1, true -} - -func findEmailEnd(s string, start int) (_ int, _ bool) { - end := start - allowDot := false -loop: - for end < len(s) { - b := s[end] - switch { - case emailcs[b]: - allowDot = true - case b == '.': - if !allowDot { - return - } - allowDot = false - case b == '@': - break loop - default: - return - } - end++ - } - if end >= len(s)-5 { - return - } - if end > start && s[end-1] == '.' { - return - } - - var dot int - var ok bool - end, dot, ok = findHostnameEnd(s, end+1) - if !ok || dot == -1 { - return - } - - if dot+5 <= len(s) && s[dot+1:dot+5] == "xn--" { - return end, true - } - - if length := match(s[dot+1:]); dot+length+1 != end { - return - } - - return end, true -} diff --git a/vendor/github.com/golang-commonmark/markdown/linkify/generated.go b/vendor/github.com/golang-commonmark/markdown/linkify/generated.go deleted file mode 100644 index 4a4bfc9e..00000000 --- a/vendor/github.com/golang-commonmark/markdown/linkify/generated.go +++ /dev/null @@ -1,18175 +0,0 @@ -package linkify - -import "github.com/golang-commonmark/markdown/byteutil" - -func match(s string) int { - st := 0 - length := -1 - - for i := 0; i < len(s); i++ { - b := s[i] - - switch st { - case 0: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1 - case 'b': - st = 148 - case 'c': - st = 303 - case 'd': - st = 571 - case 'e': - st = 686 - case 'f': - st = 797 - case 'g': - st = 916 - case 'h': - st = 1014 - case 'i': - st = 1099 - case 'j': - st = 1180 - case 'k': - st = 1210 - case 'l': - st = 1253 - case 'm': - st = 1349 - case 'n': - st = 1478 - case 'o': - st = 1536 - case 'p': - st = 1577 - case 'q': - st = 1699 - case 'r': - st = 1709 - case 's': - st = 1797 - case 't': - st = 2010 - case 'u': - st = 2128 - case 'v': - st = 2147 - case 'w': - st = 2220 - case 'x': - st = 2283 - case 'y': - st = 2297 - case 'z': - st = 2330 - case '0': - st = 2348 - case '1': - st = 2350 - case '2': - st = 2352 - case '3': - st = 2354 - case '4': - st = 2356 - case '5': - st = 2358 - case '6': - st = 2360 - case '7': - st = 2362 - case '8': - st = 2364 - case '9': - st = 2366 - default: - return length - } - - case 1: - switch byteutil.ByteToLower(b) { - case 'b': - st = 2 - case 'c': - length = i + 1 - st = 12 - case 'd': - length = i + 1 - st = 39 - case 'e': - length = i + 1 - st = 44 - case 'f': - length = i + 1 - st = 47 - case 'g': - length = i + 1 - st = 49 - case 'i': - length = i + 1 - st = 54 - case 'l': - length = i + 1 - st = 62 - case 'm': - length = i + 1 - st = 74 - case 'n': - length = i + 1 - st = 82 - case 'o': - length = i + 1 - st = 88 - case 'p': - st = 89 - case 'q': - length = i + 1 - st = 98 - case 'r': - length = i + 1 - st = 106 - case 's': - length = i + 1 - st = 114 - case 't': - length = i + 1 - st = 125 - case 'u': - length = i + 1 - st = 132 - case 'w': - length = i + 1 - st = 144 - case 'x': - length = i + 1 - st = 145 - case 'z': - length = i + 1 - st = 147 - default: - return length - } - - case 2: - switch byteutil.ByteToLower(b) { - case 'b': - length = i + 1 - st = 3 - case 'o': - st = 7 - default: - return length - } - - case 3: - switch byteutil.ByteToLower(b) { - case 'o': - st = 4 - default: - return length - } - - case 4: - switch byteutil.ByteToLower(b) { - case 't': - st = 5 - default: - return length - } - - case 5: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 6 - default: - return length - } - - case 7: - switch byteutil.ByteToLower(b) { - case 'g': - st = 8 - default: - return length - } - - case 8: - switch byteutil.ByteToLower(b) { - case 'a': - st = 9 - default: - return length - } - - case 9: - switch byteutil.ByteToLower(b) { - case 'd': - st = 10 - default: - return length - } - - case 10: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 11 - default: - return length - } - - case 12: - switch byteutil.ByteToLower(b) { - case 'a': - st = 13 - case 'c': - st = 18 - case 't': - st = 33 - default: - return length - } - - case 13: - switch byteutil.ByteToLower(b) { - case 'd': - st = 14 - default: - return length - } - - case 14: - switch byteutil.ByteToLower(b) { - case 'e': - st = 15 - default: - return length - } - - case 15: - switch byteutil.ByteToLower(b) { - case 'm': - st = 16 - default: - return length - } - - case 16: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 17 - default: - return length - } - - case 18: - switch byteutil.ByteToLower(b) { - case 'e': - st = 19 - case 'o': - st = 25 - default: - return length - } - - case 19: - switch byteutil.ByteToLower(b) { - case 'n': - st = 20 - default: - return length - } - - case 20: - switch byteutil.ByteToLower(b) { - case 't': - st = 21 - default: - return length - } - - case 21: - switch byteutil.ByteToLower(b) { - case 'u': - st = 22 - default: - return length - } - - case 22: - switch byteutil.ByteToLower(b) { - case 'r': - st = 23 - default: - return length - } - - case 23: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 24 - default: - return length - } - - case 25: - switch byteutil.ByteToLower(b) { - case 'u': - st = 26 - default: - return length - } - - case 26: - switch byteutil.ByteToLower(b) { - case 'n': - st = 27 - default: - return length - } - - case 27: - switch byteutil.ByteToLower(b) { - case 't': - st = 28 - default: - return length - } - - case 28: - switch byteutil.ByteToLower(b) { - case 'a': - st = 29 - default: - return length - } - - case 29: - switch byteutil.ByteToLower(b) { - case 'n': - st = 30 - default: - return length - } - - case 30: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 31 - default: - return length - } - - case 31: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 32 - default: - return length - } - - case 33: - switch byteutil.ByteToLower(b) { - case 'i': - st = 34 - case 'o': - st = 37 - default: - return length - } - - case 34: - switch byteutil.ByteToLower(b) { - case 'v': - st = 35 - default: - return length - } - - case 35: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 36 - default: - return length - } - - case 37: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 38 - default: - return length - } - - case 39: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 40 - case 'u': - st = 41 - default: - return length - } - - case 41: - switch byteutil.ByteToLower(b) { - case 'l': - st = 42 - default: - return length - } - - case 42: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 43 - default: - return length - } - - case 44: - switch byteutil.ByteToLower(b) { - case 'r': - st = 45 - default: - return length - } - - case 45: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 46 - default: - return length - } - - case 47: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 48 - default: - return length - } - - case 49: - switch byteutil.ByteToLower(b) { - case 'e': - st = 50 - default: - return length - } - - case 50: - switch byteutil.ByteToLower(b) { - case 'n': - st = 51 - default: - return length - } - - case 51: - switch byteutil.ByteToLower(b) { - case 'c': - st = 52 - default: - return length - } - - case 52: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 53 - default: - return length - } - - case 54: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 55 - case 'r': - st = 56 - default: - return length - } - - case 56: - switch byteutil.ByteToLower(b) { - case 'f': - st = 57 - default: - return length - } - - case 57: - switch byteutil.ByteToLower(b) { - case 'o': - st = 58 - default: - return length - } - - case 58: - switch byteutil.ByteToLower(b) { - case 'r': - st = 59 - default: - return length - } - - case 59: - switch byteutil.ByteToLower(b) { - case 'c': - st = 60 - default: - return length - } - - case 60: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 61 - default: - return length - } - - case 62: - switch byteutil.ByteToLower(b) { - case 'l': - st = 63 - case 's': - st = 70 - default: - return length - } - - case 63: - switch byteutil.ByteToLower(b) { - case 'f': - st = 64 - default: - return length - } - - case 64: - switch byteutil.ByteToLower(b) { - case 'i': - st = 65 - default: - return length - } - - case 65: - switch byteutil.ByteToLower(b) { - case 'n': - st = 66 - default: - return length - } - - case 66: - switch byteutil.ByteToLower(b) { - case 'a': - st = 67 - default: - return length - } - - case 67: - switch byteutil.ByteToLower(b) { - case 'n': - st = 68 - default: - return length - } - - case 68: - switch byteutil.ByteToLower(b) { - case 'z': - length = i + 1 - st = 69 - default: - return length - } - - case 70: - switch byteutil.ByteToLower(b) { - case 'a': - st = 71 - default: - return length - } - - case 71: - switch byteutil.ByteToLower(b) { - case 'c': - st = 72 - default: - return length - } - - case 72: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 73 - default: - return length - } - - case 74: - switch byteutil.ByteToLower(b) { - case 's': - st = 75 - default: - return length - } - - case 75: - switch byteutil.ByteToLower(b) { - case 't': - st = 76 - default: - return length - } - - case 76: - switch byteutil.ByteToLower(b) { - case 'e': - st = 77 - default: - return length - } - - case 77: - switch byteutil.ByteToLower(b) { - case 'r': - st = 78 - default: - return length - } - - case 78: - switch byteutil.ByteToLower(b) { - case 'd': - st = 79 - default: - return length - } - - case 79: - switch byteutil.ByteToLower(b) { - case 'a': - st = 80 - default: - return length - } - - case 80: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 81 - default: - return length - } - - case 82: - switch byteutil.ByteToLower(b) { - case 'd': - st = 83 - default: - return length - } - - case 83: - switch byteutil.ByteToLower(b) { - case 'r': - st = 84 - default: - return length - } - - case 84: - switch byteutil.ByteToLower(b) { - case 'o': - st = 85 - default: - return length - } - - case 85: - switch byteutil.ByteToLower(b) { - case 'i': - st = 86 - default: - return length - } - - case 86: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 87 - default: - return length - } - - case 89: - switch byteutil.ByteToLower(b) { - case 'a': - st = 90 - default: - return length - } - - case 90: - switch byteutil.ByteToLower(b) { - case 'r': - st = 91 - default: - return length - } - - case 91: - switch byteutil.ByteToLower(b) { - case 't': - st = 92 - default: - return length - } - - case 92: - switch byteutil.ByteToLower(b) { - case 'm': - st = 93 - default: - return length - } - - case 93: - switch byteutil.ByteToLower(b) { - case 'e': - st = 94 - default: - return length - } - - case 94: - switch byteutil.ByteToLower(b) { - case 'n': - st = 95 - default: - return length - } - - case 95: - switch byteutil.ByteToLower(b) { - case 't': - st = 96 - default: - return length - } - - case 96: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 97 - default: - return length - } - - case 98: - switch byteutil.ByteToLower(b) { - case 'u': - st = 99 - default: - return length - } - - case 99: - switch byteutil.ByteToLower(b) { - case 'a': - st = 100 - default: - return length - } - - case 100: - switch byteutil.ByteToLower(b) { - case 'r': - st = 101 - default: - return length - } - - case 101: - switch byteutil.ByteToLower(b) { - case 'e': - st = 102 - default: - return length - } - - case 102: - switch byteutil.ByteToLower(b) { - case 'l': - st = 103 - default: - return length - } - - case 103: - switch byteutil.ByteToLower(b) { - case 'l': - st = 104 - default: - return length - } - - case 104: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 105 - default: - return length - } - - case 106: - switch byteutil.ByteToLower(b) { - case 'c': - st = 107 - case 'm': - st = 110 - case 'p': - st = 112 - default: - return length - } - - case 107: - switch byteutil.ByteToLower(b) { - case 'h': - st = 108 - default: - return length - } - - case 108: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 109 - default: - return length - } - - case 110: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 111 - default: - return length - } - - case 112: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 113 - default: - return length - } - - case 114: - switch byteutil.ByteToLower(b) { - case 'i': - st = 115 - case 's': - st = 117 - default: - return length - } - - case 115: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 116 - default: - return length - } - - case 117: - switch byteutil.ByteToLower(b) { - case 'o': - st = 118 - default: - return length - } - - case 118: - switch byteutil.ByteToLower(b) { - case 'c': - st = 119 - default: - return length - } - - case 119: - switch byteutil.ByteToLower(b) { - case 'i': - st = 120 - default: - return length - } - - case 120: - switch byteutil.ByteToLower(b) { - case 'a': - st = 121 - default: - return length - } - - case 121: - switch byteutil.ByteToLower(b) { - case 't': - st = 122 - default: - return length - } - - case 122: - switch byteutil.ByteToLower(b) { - case 'e': - st = 123 - default: - return length - } - - case 123: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 124 - default: - return length - } - - case 125: - switch byteutil.ByteToLower(b) { - case 't': - st = 126 - default: - return length - } - - case 126: - switch byteutil.ByteToLower(b) { - case 'o': - st = 127 - default: - return length - } - - case 127: - switch byteutil.ByteToLower(b) { - case 'r': - st = 128 - default: - return length - } - - case 128: - switch byteutil.ByteToLower(b) { - case 'n': - st = 129 - default: - return length - } - - case 129: - switch byteutil.ByteToLower(b) { - case 'e': - st = 130 - default: - return length - } - - case 130: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 131 - default: - return length - } - - case 132: - switch byteutil.ByteToLower(b) { - case 'c': - st = 133 - case 'd': - st = 138 - case 't': - st = 141 - default: - return length - } - - case 133: - switch byteutil.ByteToLower(b) { - case 't': - st = 134 - default: - return length - } - - case 134: - switch byteutil.ByteToLower(b) { - case 'i': - st = 135 - default: - return length - } - - case 135: - switch byteutil.ByteToLower(b) { - case 'o': - st = 136 - default: - return length - } - - case 136: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 137 - default: - return length - } - - case 138: - switch byteutil.ByteToLower(b) { - case 'i': - st = 139 - default: - return length - } - - case 139: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 140 - default: - return length - } - - case 141: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 142 - default: - return length - } - - case 142: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 143 - default: - return length - } - - case 145: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 146 - default: - return length - } - - case 148: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 149 - case 'b': - length = i + 1 - st = 177 - case 'd': - length = i + 1 - st = 181 - case 'e': - length = i + 1 - st = 182 - case 'f': - length = i + 1 - st = 191 - case 'g': - length = i + 1 - st = 192 - case 'h': - length = i + 1 - st = 193 - case 'i': - length = i + 1 - st = 194 - case 'j': - length = i + 1 - st = 207 - case 'l': - st = 208 - case 'm': - length = i + 1 - st = 227 - case 'n': - length = i + 1 - st = 229 - case 'o': - length = i + 1 - st = 238 - case 'r': - length = i + 1 - st = 251 - case 's': - length = i + 1 - st = 275 - case 't': - length = i + 1 - st = 276 - case 'u': - st = 277 - case 'v': - length = i + 1 - st = 298 - case 'w': - length = i + 1 - st = 299 - case 'y': - length = i + 1 - st = 300 - case 'z': - length = i + 1 - st = 301 - default: - return length - } - - case 149: - switch byteutil.ByteToLower(b) { - case 'n': - st = 150 - case 'r': - length = i + 1 - st = 153 - case 'u': - st = 168 - case 'y': - st = 173 - default: - return length - } - - case 150: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 151 - case 'k': - length = i + 1 - st = 152 - default: - return length - } - - case 153: - switch byteutil.ByteToLower(b) { - case 'c': - st = 154 - case 'g': - st = 163 - default: - return length - } - - case 154: - switch byteutil.ByteToLower(b) { - case 'l': - st = 155 - default: - return length - } - - case 155: - switch byteutil.ByteToLower(b) { - case 'a': - st = 156 - default: - return length - } - - case 156: - switch byteutil.ByteToLower(b) { - case 'y': - st = 157 - default: - return length - } - - case 157: - switch byteutil.ByteToLower(b) { - case 'c': - st = 158 - case 's': - length = i + 1 - st = 162 - default: - return length - } - - case 158: - switch byteutil.ByteToLower(b) { - case 'a': - st = 159 - default: - return length - } - - case 159: - switch byteutil.ByteToLower(b) { - case 'r': - st = 160 - default: - return length - } - - case 160: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 161 - default: - return length - } - - case 163: - switch byteutil.ByteToLower(b) { - case 'a': - st = 164 - default: - return length - } - - case 164: - switch byteutil.ByteToLower(b) { - case 'i': - st = 165 - default: - return length - } - - case 165: - switch byteutil.ByteToLower(b) { - case 'n': - st = 166 - default: - return length - } - - case 166: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 167 - default: - return length - } - - case 168: - switch byteutil.ByteToLower(b) { - case 'h': - st = 169 - default: - return length - } - - case 169: - switch byteutil.ByteToLower(b) { - case 'a': - st = 170 - default: - return length - } - - case 170: - switch byteutil.ByteToLower(b) { - case 'u': - st = 171 - default: - return length - } - - case 171: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 172 - default: - return length - } - - case 173: - switch byteutil.ByteToLower(b) { - case 'e': - st = 174 - default: - return length - } - - case 174: - switch byteutil.ByteToLower(b) { - case 'r': - st = 175 - default: - return length - } - - case 175: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 176 - default: - return length - } - - case 177: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 178 - case 'v': - st = 179 - default: - return length - } - - case 179: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 180 - default: - return length - } - - case 182: - switch byteutil.ByteToLower(b) { - case 'e': - st = 183 - case 'r': - st = 185 - case 's': - st = 189 - default: - return length - } - - case 183: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 184 - default: - return length - } - - case 185: - switch byteutil.ByteToLower(b) { - case 'l': - st = 186 - default: - return length - } - - case 186: - switch byteutil.ByteToLower(b) { - case 'i': - st = 187 - default: - return length - } - - case 187: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 188 - default: - return length - } - - case 189: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 190 - default: - return length - } - - case 194: - switch byteutil.ByteToLower(b) { - case 'b': - st = 195 - case 'd': - length = i + 1 - st = 198 - case 'k': - st = 199 - case 'n': - st = 201 - case 'o': - length = i + 1 - st = 204 - case 't': - length = i + 1 - st = 205 - case 'z': - length = i + 1 - st = 206 - default: - return length - } - - case 195: - switch byteutil.ByteToLower(b) { - case 'l': - st = 196 - default: - return length - } - - case 196: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 197 - default: - return length - } - - case 199: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 200 - default: - return length - } - - case 201: - switch byteutil.ByteToLower(b) { - case 'g': - st = 202 - default: - return length - } - - case 202: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 203 - default: - return length - } - - case 208: - switch byteutil.ByteToLower(b) { - case 'a': - st = 209 - case 'o': - st = 218 - case 'u': - st = 225 - default: - return length - } - - case 209: - switch byteutil.ByteToLower(b) { - case 'c': - st = 210 - default: - return length - } - - case 210: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 211 - default: - return length - } - - case 211: - switch byteutil.ByteToLower(b) { - case 'f': - st = 212 - default: - return length - } - - case 212: - switch byteutil.ByteToLower(b) { - case 'r': - st = 213 - default: - return length - } - - case 213: - switch byteutil.ByteToLower(b) { - case 'i': - st = 214 - default: - return length - } - - case 214: - switch byteutil.ByteToLower(b) { - case 'd': - st = 215 - default: - return length - } - - case 215: - switch byteutil.ByteToLower(b) { - case 'a': - st = 216 - default: - return length - } - - case 216: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 217 - default: - return length - } - - case 218: - switch byteutil.ByteToLower(b) { - case 'o': - st = 219 - default: - return length - } - - case 219: - switch byteutil.ByteToLower(b) { - case 'm': - st = 220 - default: - return length - } - - case 220: - switch byteutil.ByteToLower(b) { - case 'b': - st = 221 - default: - return length - } - - case 221: - switch byteutil.ByteToLower(b) { - case 'e': - st = 222 - default: - return length - } - - case 222: - switch byteutil.ByteToLower(b) { - case 'r': - st = 223 - default: - return length - } - - case 223: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 224 - default: - return length - } - - case 225: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 226 - default: - return length - } - - case 227: - switch byteutil.ByteToLower(b) { - case 'w': - length = i + 1 - st = 228 - default: - return length - } - - case 229: - switch byteutil.ByteToLower(b) { - case 'p': - st = 230 - default: - return length - } - - case 230: - switch byteutil.ByteToLower(b) { - case 'p': - st = 231 - default: - return length - } - - case 231: - switch byteutil.ByteToLower(b) { - case 'a': - st = 232 - default: - return length - } - - case 232: - switch byteutil.ByteToLower(b) { - case 'r': - st = 233 - default: - return length - } - - case 233: - switch byteutil.ByteToLower(b) { - case 'i': - st = 234 - default: - return length - } - - case 234: - switch byteutil.ByteToLower(b) { - case 'b': - st = 235 - default: - return length - } - - case 235: - switch byteutil.ByteToLower(b) { - case 'a': - st = 236 - default: - return length - } - - case 236: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 237 - default: - return length - } - - case 238: - switch byteutil.ByteToLower(b) { - case 'a': - st = 239 - case 'n': - st = 242 - case 'o': - length = i + 1 - st = 244 - case 'u': - st = 245 - default: - return length - } - - case 239: - switch byteutil.ByteToLower(b) { - case 't': - st = 240 - default: - return length - } - - case 240: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 241 - default: - return length - } - - case 242: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 243 - default: - return length - } - - case 245: - switch byteutil.ByteToLower(b) { - case 't': - st = 246 - default: - return length - } - - case 246: - switch byteutil.ByteToLower(b) { - case 'i': - st = 247 - default: - return length - } - - case 247: - switch byteutil.ByteToLower(b) { - case 'q': - st = 248 - default: - return length - } - - case 248: - switch byteutil.ByteToLower(b) { - case 'u': - st = 249 - default: - return length - } - - case 249: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 250 - default: - return length - } - - case 251: - switch byteutil.ByteToLower(b) { - case 'i': - st = 252 - case 'o': - st = 261 - case 'u': - st = 269 - default: - return length - } - - case 252: - switch byteutil.ByteToLower(b) { - case 'd': - st = 253 - default: - return length - } - - case 253: - switch byteutil.ByteToLower(b) { - case 'g': - st = 254 - default: - return length - } - - case 254: - switch byteutil.ByteToLower(b) { - case 'e': - st = 255 - default: - return length - } - - case 255: - switch byteutil.ByteToLower(b) { - case 's': - st = 256 - default: - return length - } - - case 256: - switch byteutil.ByteToLower(b) { - case 't': - st = 257 - default: - return length - } - - case 257: - switch byteutil.ByteToLower(b) { - case 'o': - st = 258 - default: - return length - } - - case 258: - switch byteutil.ByteToLower(b) { - case 'n': - st = 259 - default: - return length - } - - case 259: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 260 - default: - return length - } - - case 261: - switch byteutil.ByteToLower(b) { - case 'k': - st = 262 - case 't': - st = 265 - default: - return length - } - - case 262: - switch byteutil.ByteToLower(b) { - case 'e': - st = 263 - default: - return length - } - - case 263: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 264 - default: - return length - } - - case 265: - switch byteutil.ByteToLower(b) { - case 'h': - st = 266 - default: - return length - } - - case 266: - switch byteutil.ByteToLower(b) { - case 'e': - st = 267 - default: - return length - } - - case 267: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 268 - default: - return length - } - - case 269: - switch byteutil.ByteToLower(b) { - case 's': - st = 270 - default: - return length - } - - case 270: - switch byteutil.ByteToLower(b) { - case 's': - st = 271 - default: - return length - } - - case 271: - switch byteutil.ByteToLower(b) { - case 'e': - st = 272 - default: - return length - } - - case 272: - switch byteutil.ByteToLower(b) { - case 'l': - st = 273 - default: - return length - } - - case 273: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 274 - default: - return length - } - - case 277: - switch byteutil.ByteToLower(b) { - case 'd': - st = 278 - case 'i': - st = 284 - case 's': - st = 290 - case 'z': - st = 296 - default: - return length - } - - case 278: - switch byteutil.ByteToLower(b) { - case 'a': - st = 279 - default: - return length - } - - case 279: - switch byteutil.ByteToLower(b) { - case 'p': - st = 280 - default: - return length - } - - case 280: - switch byteutil.ByteToLower(b) { - case 'e': - st = 281 - default: - return length - } - - case 281: - switch byteutil.ByteToLower(b) { - case 's': - st = 282 - default: - return length - } - - case 282: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 283 - default: - return length - } - - case 284: - switch byteutil.ByteToLower(b) { - case 'l': - st = 285 - default: - return length - } - - case 285: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 286 - default: - return length - } - - case 286: - switch byteutil.ByteToLower(b) { - case 'e': - st = 287 - default: - return length - } - - case 287: - switch byteutil.ByteToLower(b) { - case 'r': - st = 288 - default: - return length - } - - case 288: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 289 - default: - return length - } - - case 290: - switch byteutil.ByteToLower(b) { - case 'i': - st = 291 - default: - return length - } - - case 291: - switch byteutil.ByteToLower(b) { - case 'n': - st = 292 - default: - return length - } - - case 292: - switch byteutil.ByteToLower(b) { - case 'e': - st = 293 - default: - return length - } - - case 293: - switch byteutil.ByteToLower(b) { - case 's': - st = 294 - default: - return length - } - - case 294: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 295 - default: - return length - } - - case 296: - switch byteutil.ByteToLower(b) { - case 'z': - length = i + 1 - st = 297 - default: - return length - } - - case 301: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 302 - default: - return length - } - - case 303: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 304 - case 'b': - st = 366 - case 'c': - length = i + 1 - st = 368 - case 'd': - length = i + 1 - st = 369 - case 'e': - st = 370 - case 'f': - length = i + 1 - st = 378 - case 'g': - length = i + 1 - st = 381 - case 'h': - length = i + 1 - st = 382 - case 'i': - length = i + 1 - st = 409 - case 'k': - length = i + 1 - st = 417 - case 'l': - length = i + 1 - st = 418 - case 'm': - length = i + 1 - st = 443 - case 'n': - length = i + 1 - st = 444 - case 'o': - length = i + 1 - st = 445 - case 'r': - length = i + 1 - st = 532 - case 'u': - length = i + 1 - st = 552 - case 'v': - length = i + 1 - st = 561 - case 'w': - length = i + 1 - st = 562 - case 'x': - length = i + 1 - st = 563 - case 'y': - length = i + 1 - st = 564 - case 'z': - length = i + 1 - st = 570 - default: - return length - } - - case 304: - switch byteutil.ByteToLower(b) { - case 'b': - length = i + 1 - st = 305 - case 'f': - st = 306 - case 'l': - length = i + 1 - st = 308 - case 'm': - st = 309 - case 'n': - st = 314 - case 'p': - st = 328 - case 'r': - st = 338 - case 's': - st = 354 - case 't': - length = i + 1 - st = 360 - default: - return length - } - - case 306: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 307 - default: - return length - } - - case 309: - switch byteutil.ByteToLower(b) { - case 'e': - st = 310 - case 'p': - length = i + 1 - st = 313 - default: - return length - } - - case 310: - switch byteutil.ByteToLower(b) { - case 'r': - st = 311 - default: - return length - } - - case 311: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 312 - default: - return length - } - - case 314: - switch byteutil.ByteToLower(b) { - case 'c': - st = 315 - case 'o': - st = 326 - default: - return length - } - - case 315: - switch byteutil.ByteToLower(b) { - case 'e': - st = 316 - default: - return length - } - - case 316: - switch byteutil.ByteToLower(b) { - case 'r': - st = 317 - default: - return length - } - - case 317: - switch byteutil.ByteToLower(b) { - case 'r': - st = 318 - default: - return length - } - - case 318: - switch byteutil.ByteToLower(b) { - case 'e': - st = 319 - default: - return length - } - - case 319: - switch byteutil.ByteToLower(b) { - case 's': - st = 320 - default: - return length - } - - case 320: - switch byteutil.ByteToLower(b) { - case 'e': - st = 321 - default: - return length - } - - case 321: - switch byteutil.ByteToLower(b) { - case 'a': - st = 322 - default: - return length - } - - case 322: - switch byteutil.ByteToLower(b) { - case 'r': - st = 323 - default: - return length - } - - case 323: - switch byteutil.ByteToLower(b) { - case 'c': - st = 324 - default: - return length - } - - case 324: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 325 - default: - return length - } - - case 326: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 327 - default: - return length - } - - case 328: - switch byteutil.ByteToLower(b) { - case 'e': - st = 329 - case 'i': - st = 334 - default: - return length - } - - case 329: - switch byteutil.ByteToLower(b) { - case 't': - st = 330 - default: - return length - } - - case 330: - switch byteutil.ByteToLower(b) { - case 'o': - st = 331 - default: - return length - } - - case 331: - switch byteutil.ByteToLower(b) { - case 'w': - st = 332 - default: - return length - } - - case 332: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 333 - default: - return length - } - - case 334: - switch byteutil.ByteToLower(b) { - case 't': - st = 335 - default: - return length - } - - case 335: - switch byteutil.ByteToLower(b) { - case 'a': - st = 336 - default: - return length - } - - case 336: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 337 - default: - return length - } - - case 338: - switch byteutil.ByteToLower(b) { - case 'a': - st = 339 - case 'd': - st = 343 - case 'e': - length = i + 1 - st = 345 - case 's': - length = i + 1 - st = 349 - case 't': - st = 350 - default: - return length - } - - case 339: - switch byteutil.ByteToLower(b) { - case 'v': - st = 340 - default: - return length - } - - case 340: - switch byteutil.ByteToLower(b) { - case 'a': - st = 341 - default: - return length - } - - case 341: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 342 - default: - return length - } - - case 343: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 344 - default: - return length - } - - case 345: - switch byteutil.ByteToLower(b) { - case 'e': - st = 346 - default: - return length - } - - case 346: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 347 - default: - return length - } - - case 347: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 348 - default: - return length - } - - case 350: - switch byteutil.ByteToLower(b) { - case 'i': - st = 351 - default: - return length - } - - case 351: - switch byteutil.ByteToLower(b) { - case 'e': - st = 352 - default: - return length - } - - case 352: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 353 - default: - return length - } - - case 354: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 355 - case 'h': - length = i + 1 - st = 356 - case 'i': - st = 357 - default: - return length - } - - case 357: - switch byteutil.ByteToLower(b) { - case 'n': - st = 358 - default: - return length - } - - case 358: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 359 - default: - return length - } - - case 360: - switch byteutil.ByteToLower(b) { - case 'e': - st = 361 - default: - return length - } - - case 361: - switch byteutil.ByteToLower(b) { - case 'r': - st = 362 - default: - return length - } - - case 362: - switch byteutil.ByteToLower(b) { - case 'i': - st = 363 - default: - return length - } - - case 363: - switch byteutil.ByteToLower(b) { - case 'n': - st = 364 - default: - return length - } - - case 364: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 365 - default: - return length - } - - case 366: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 367 - default: - return length - } - - case 370: - switch byteutil.ByteToLower(b) { - case 'n': - st = 371 - case 'o': - length = i + 1 - st = 375 - case 'r': - st = 376 - default: - return length - } - - case 371: - switch byteutil.ByteToLower(b) { - case 't': - st = 372 - default: - return length - } - - case 372: - switch byteutil.ByteToLower(b) { - case 'e': - st = 373 - default: - return length - } - - case 373: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 374 - default: - return length - } - - case 376: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 377 - default: - return length - } - - case 378: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 379 - case 'd': - length = i + 1 - st = 380 - default: - return length - } - - case 382: - switch byteutil.ByteToLower(b) { - case 'a': - st = 383 - case 'e': - st = 389 - case 'l': - st = 392 - case 'r': - st = 395 - case 'u': - st = 405 - default: - return length - } - - case 383: - switch byteutil.ByteToLower(b) { - case 'n': - st = 384 - case 't': - length = i + 1 - st = 388 - default: - return length - } - - case 384: - switch byteutil.ByteToLower(b) { - case 'n': - st = 385 - default: - return length - } - - case 385: - switch byteutil.ByteToLower(b) { - case 'e': - st = 386 - default: - return length - } - - case 386: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 387 - default: - return length - } - - case 389: - switch byteutil.ByteToLower(b) { - case 'a': - st = 390 - default: - return length - } - - case 390: - switch byteutil.ByteToLower(b) { - case 'p': - length = i + 1 - st = 391 - default: - return length - } - - case 392: - switch byteutil.ByteToLower(b) { - case 'o': - st = 393 - default: - return length - } - - case 393: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 394 - default: - return length - } - - case 395: - switch byteutil.ByteToLower(b) { - case 'i': - st = 396 - case 'o': - st = 402 - default: - return length - } - - case 396: - switch byteutil.ByteToLower(b) { - case 's': - st = 397 - default: - return length - } - - case 397: - switch byteutil.ByteToLower(b) { - case 't': - st = 398 - default: - return length - } - - case 398: - switch byteutil.ByteToLower(b) { - case 'm': - st = 399 - default: - return length - } - - case 399: - switch byteutil.ByteToLower(b) { - case 'a': - st = 400 - default: - return length - } - - case 400: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 401 - default: - return length - } - - case 402: - switch byteutil.ByteToLower(b) { - case 'm': - st = 403 - default: - return length - } - - case 403: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 404 - default: - return length - } - - case 405: - switch byteutil.ByteToLower(b) { - case 'r': - st = 406 - default: - return length - } - - case 406: - switch byteutil.ByteToLower(b) { - case 'c': - st = 407 - default: - return length - } - - case 407: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 408 - default: - return length - } - - case 409: - switch byteutil.ByteToLower(b) { - case 's': - st = 410 - case 't': - st = 413 - default: - return length - } - - case 410: - switch byteutil.ByteToLower(b) { - case 'c': - st = 411 - default: - return length - } - - case 411: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 412 - default: - return length - } - - case 413: - switch byteutil.ByteToLower(b) { - case 'i': - st = 414 - case 'y': - length = i + 1 - st = 416 - default: - return length - } - - case 414: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 415 - default: - return length - } - - case 418: - switch byteutil.ByteToLower(b) { - case 'a': - st = 419 - case 'e': - st = 423 - case 'i': - st = 429 - case 'o': - st = 435 - case 'u': - st = 441 - default: - return length - } - - case 419: - switch byteutil.ByteToLower(b) { - case 'i': - st = 420 - default: - return length - } - - case 420: - switch byteutil.ByteToLower(b) { - case 'm': - st = 421 - default: - return length - } - - case 421: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 422 - default: - return length - } - - case 423: - switch byteutil.ByteToLower(b) { - case 'a': - st = 424 - default: - return length - } - - case 424: - switch byteutil.ByteToLower(b) { - case 'n': - st = 425 - default: - return length - } - - case 425: - switch byteutil.ByteToLower(b) { - case 'i': - st = 426 - default: - return length - } - - case 426: - switch byteutil.ByteToLower(b) { - case 'n': - st = 427 - default: - return length - } - - case 427: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 428 - default: - return length - } - - case 429: - switch byteutil.ByteToLower(b) { - case 'c': - st = 430 - case 'n': - st = 432 - default: - return length - } - - case 430: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 431 - default: - return length - } - - case 432: - switch byteutil.ByteToLower(b) { - case 'i': - st = 433 - default: - return length - } - - case 433: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 434 - default: - return length - } - - case 435: - switch byteutil.ByteToLower(b) { - case 't': - st = 436 - default: - return length - } - - case 436: - switch byteutil.ByteToLower(b) { - case 'h': - st = 437 - default: - return length - } - - case 437: - switch byteutil.ByteToLower(b) { - case 'i': - st = 438 - default: - return length - } - - case 438: - switch byteutil.ByteToLower(b) { - case 'n': - st = 439 - default: - return length - } - - case 439: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 440 - default: - return length - } - - case 441: - switch byteutil.ByteToLower(b) { - case 'b': - length = i + 1 - st = 442 - default: - return length - } - - case 445: - switch byteutil.ByteToLower(b) { - case 'a': - st = 446 - case 'd': - st = 449 - case 'f': - st = 452 - case 'l': - st = 456 - case 'm': - length = i + 1 - st = 465 - case 'n': - st = 480 - case 'o': - st = 507 - case 'r': - st = 514 - case 'u': - st = 519 - default: - return length - } - - case 446: - switch byteutil.ByteToLower(b) { - case 'c': - st = 447 - default: - return length - } - - case 447: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 448 - default: - return length - } - - case 449: - switch byteutil.ByteToLower(b) { - case 'e': - st = 450 - default: - return length - } - - case 450: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 451 - default: - return length - } - - case 452: - switch byteutil.ByteToLower(b) { - case 'f': - st = 453 - default: - return length - } - - case 453: - switch byteutil.ByteToLower(b) { - case 'e': - st = 454 - default: - return length - } - - case 454: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 455 - default: - return length - } - - case 456: - switch byteutil.ByteToLower(b) { - case 'l': - st = 457 - case 'o': - st = 461 - default: - return length - } - - case 457: - switch byteutil.ByteToLower(b) { - case 'e': - st = 458 - default: - return length - } - - case 458: - switch byteutil.ByteToLower(b) { - case 'g': - st = 459 - default: - return length - } - - case 459: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 460 - default: - return length - } - - case 461: - switch byteutil.ByteToLower(b) { - case 'g': - st = 462 - default: - return length - } - - case 462: - switch byteutil.ByteToLower(b) { - case 'n': - st = 463 - default: - return length - } - - case 463: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 464 - default: - return length - } - - case 465: - switch byteutil.ByteToLower(b) { - case 'm': - st = 466 - case 'p': - st = 472 - default: - return length - } - - case 466: - switch byteutil.ByteToLower(b) { - case 'u': - st = 467 - default: - return length - } - - case 467: - switch byteutil.ByteToLower(b) { - case 'n': - st = 468 - default: - return length - } - - case 468: - switch byteutil.ByteToLower(b) { - case 'i': - st = 469 - default: - return length - } - - case 469: - switch byteutil.ByteToLower(b) { - case 't': - st = 470 - default: - return length - } - - case 470: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 471 - default: - return length - } - - case 472: - switch byteutil.ByteToLower(b) { - case 'a': - st = 473 - case 'u': - st = 476 - default: - return length - } - - case 473: - switch byteutil.ByteToLower(b) { - case 'n': - st = 474 - default: - return length - } - - case 474: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 475 - default: - return length - } - - case 476: - switch byteutil.ByteToLower(b) { - case 't': - st = 477 - default: - return length - } - - case 477: - switch byteutil.ByteToLower(b) { - case 'e': - st = 478 - default: - return length - } - - case 478: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 479 - default: - return length - } - - case 480: - switch byteutil.ByteToLower(b) { - case 'd': - st = 481 - case 's': - st = 484 - case 't': - st = 499 - default: - return length - } - - case 481: - switch byteutil.ByteToLower(b) { - case 'o': - st = 482 - default: - return length - } - - case 482: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 483 - default: - return length - } - - case 484: - switch byteutil.ByteToLower(b) { - case 't': - st = 485 - case 'u': - st = 493 - default: - return length - } - - case 485: - switch byteutil.ByteToLower(b) { - case 'r': - st = 486 - default: - return length - } - - case 486: - switch byteutil.ByteToLower(b) { - case 'u': - st = 487 - default: - return length - } - - case 487: - switch byteutil.ByteToLower(b) { - case 'c': - st = 488 - default: - return length - } - - case 488: - switch byteutil.ByteToLower(b) { - case 't': - st = 489 - default: - return length - } - - case 489: - switch byteutil.ByteToLower(b) { - case 'i': - st = 490 - default: - return length - } - - case 490: - switch byteutil.ByteToLower(b) { - case 'o': - st = 491 - default: - return length - } - - case 491: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 492 - default: - return length - } - - case 493: - switch byteutil.ByteToLower(b) { - case 'l': - st = 494 - default: - return length - } - - case 494: - switch byteutil.ByteToLower(b) { - case 't': - st = 495 - default: - return length - } - - case 495: - switch byteutil.ByteToLower(b) { - case 'i': - st = 496 - default: - return length - } - - case 496: - switch byteutil.ByteToLower(b) { - case 'n': - st = 497 - default: - return length - } - - case 497: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 498 - default: - return length - } - - case 499: - switch byteutil.ByteToLower(b) { - case 'r': - st = 500 - default: - return length - } - - case 500: - switch byteutil.ByteToLower(b) { - case 'a': - st = 501 - default: - return length - } - - case 501: - switch byteutil.ByteToLower(b) { - case 'c': - st = 502 - default: - return length - } - - case 502: - switch byteutil.ByteToLower(b) { - case 't': - st = 503 - default: - return length - } - - case 503: - switch byteutil.ByteToLower(b) { - case 'o': - st = 504 - default: - return length - } - - case 504: - switch byteutil.ByteToLower(b) { - case 'r': - st = 505 - default: - return length - } - - case 505: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 506 - default: - return length - } - - case 507: - switch byteutil.ByteToLower(b) { - case 'k': - st = 508 - case 'l': - length = i + 1 - st = 512 - case 'p': - length = i + 1 - st = 513 - default: - return length - } - - case 508: - switch byteutil.ByteToLower(b) { - case 'i': - st = 509 - default: - return length - } - - case 509: - switch byteutil.ByteToLower(b) { - case 'n': - st = 510 - default: - return length - } - - case 510: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 511 - default: - return length - } - - case 514: - switch byteutil.ByteToLower(b) { - case 's': - st = 515 - default: - return length - } - - case 515: - switch byteutil.ByteToLower(b) { - case 'i': - st = 516 - default: - return length - } - - case 516: - switch byteutil.ByteToLower(b) { - case 'c': - st = 517 - default: - return length - } - - case 517: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 518 - default: - return length - } - - case 519: - switch byteutil.ByteToLower(b) { - case 'n': - st = 520 - case 'p': - st = 524 - case 'r': - st = 528 - default: - return length - } - - case 520: - switch byteutil.ByteToLower(b) { - case 't': - st = 521 - default: - return length - } - - case 521: - switch byteutil.ByteToLower(b) { - case 'r': - st = 522 - default: - return length - } - - case 522: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 523 - default: - return length - } - - case 524: - switch byteutil.ByteToLower(b) { - case 'o': - st = 525 - default: - return length - } - - case 525: - switch byteutil.ByteToLower(b) { - case 'n': - st = 526 - default: - return length - } - - case 526: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 527 - default: - return length - } - - case 528: - switch byteutil.ByteToLower(b) { - case 's': - st = 529 - default: - return length - } - - case 529: - switch byteutil.ByteToLower(b) { - case 'e': - st = 530 - default: - return length - } - - case 530: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 531 - default: - return length - } - - case 532: - switch byteutil.ByteToLower(b) { - case 'e': - st = 533 - case 'i': - st = 541 - case 's': - length = i + 1 - st = 546 - case 'u': - st = 547 - default: - return length - } - - case 533: - switch byteutil.ByteToLower(b) { - case 'd': - st = 534 - default: - return length - } - - case 534: - switch byteutil.ByteToLower(b) { - case 'i': - st = 535 - default: - return length - } - - case 535: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 536 - default: - return length - } - - case 536: - switch byteutil.ByteToLower(b) { - case 'c': - st = 537 - default: - return length - } - - case 537: - switch byteutil.ByteToLower(b) { - case 'a': - st = 538 - default: - return length - } - - case 538: - switch byteutil.ByteToLower(b) { - case 'r': - st = 539 - default: - return length - } - - case 539: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 540 - default: - return length - } - - case 541: - switch byteutil.ByteToLower(b) { - case 'c': - st = 542 - default: - return length - } - - case 542: - switch byteutil.ByteToLower(b) { - case 'k': - st = 543 - default: - return length - } - - case 543: - switch byteutil.ByteToLower(b) { - case 'e': - st = 544 - default: - return length - } - - case 544: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 545 - default: - return length - } - - case 547: - switch byteutil.ByteToLower(b) { - case 'i': - st = 548 - default: - return length - } - - case 548: - switch byteutil.ByteToLower(b) { - case 's': - st = 549 - default: - return length - } - - case 549: - switch byteutil.ByteToLower(b) { - case 'e': - st = 550 - default: - return length - } - - case 550: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 551 - default: - return length - } - - case 552: - switch byteutil.ByteToLower(b) { - case 'i': - st = 553 - default: - return length - } - - case 553: - switch byteutil.ByteToLower(b) { - case 's': - st = 554 - default: - return length - } - - case 554: - switch byteutil.ByteToLower(b) { - case 'i': - st = 555 - default: - return length - } - - case 555: - switch byteutil.ByteToLower(b) { - case 'n': - st = 556 - default: - return length - } - - case 556: - switch byteutil.ByteToLower(b) { - case 'e': - st = 557 - default: - return length - } - - case 557: - switch byteutil.ByteToLower(b) { - case 'l': - st = 558 - default: - return length - } - - case 558: - switch byteutil.ByteToLower(b) { - case 'l': - st = 559 - default: - return length - } - - case 559: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 560 - default: - return length - } - - case 564: - switch byteutil.ByteToLower(b) { - case 'm': - st = 565 - case 'o': - st = 568 - default: - return length - } - - case 565: - switch byteutil.ByteToLower(b) { - case 'r': - st = 566 - default: - return length - } - - case 566: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 567 - default: - return length - } - - case 568: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 569 - default: - return length - } - - case 571: - switch byteutil.ByteToLower(b) { - case 'a': - st = 572 - case 'c': - st = 589 - case 'e': - length = i + 1 - st = 592 - case 'i': - st = 624 - case 'j': - length = i + 1 - st = 651 - case 'k': - length = i + 1 - st = 652 - case 'm': - length = i + 1 - st = 653 - case 'n': - st = 654 - case 'o': - length = i + 1 - st = 656 - case 'u': - st = 677 - case 'v': - st = 682 - case 'z': - length = i + 1 - st = 685 - default: - return length - } - - case 572: - switch byteutil.ByteToLower(b) { - case 'b': - st = 573 - case 'd': - length = i + 1 - st = 576 - case 'n': - st = 577 - case 't': - st = 580 - case 'y': - length = i + 1 - st = 588 - default: - return length - } - - case 573: - switch byteutil.ByteToLower(b) { - case 'u': - st = 574 - default: - return length - } - - case 574: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 575 - default: - return length - } - - case 577: - switch byteutil.ByteToLower(b) { - case 'c': - st = 578 - default: - return length - } - - case 578: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 579 - default: - return length - } - - case 580: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 581 - case 'i': - st = 582 - case 's': - st = 585 - default: - return length - } - - case 582: - switch byteutil.ByteToLower(b) { - case 'n': - st = 583 - default: - return length - } - - case 583: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 584 - default: - return length - } - - case 585: - switch byteutil.ByteToLower(b) { - case 'u': - st = 586 - default: - return length - } - - case 586: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 587 - default: - return length - } - - case 589: - switch byteutil.ByteToLower(b) { - case 'l': - st = 590 - default: - return length - } - - case 590: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 591 - default: - return length - } - - case 592: - switch byteutil.ByteToLower(b) { - case 'a': - st = 593 - case 'g': - st = 596 - case 'l': - st = 600 - case 'm': - st = 606 - case 'n': - st = 612 - case 's': - st = 619 - case 'v': - length = i + 1 - st = 623 - default: - return length - } - - case 593: - switch byteutil.ByteToLower(b) { - case 'l': - st = 594 - default: - return length - } - - case 594: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 595 - default: - return length - } - - case 596: - switch byteutil.ByteToLower(b) { - case 'r': - st = 597 - default: - return length - } - - case 597: - switch byteutil.ByteToLower(b) { - case 'e': - st = 598 - default: - return length - } - - case 598: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 599 - default: - return length - } - - case 600: - switch byteutil.ByteToLower(b) { - case 'i': - st = 601 - default: - return length - } - - case 601: - switch byteutil.ByteToLower(b) { - case 'v': - st = 602 - default: - return length - } - - case 602: - switch byteutil.ByteToLower(b) { - case 'e': - st = 603 - default: - return length - } - - case 603: - switch byteutil.ByteToLower(b) { - case 'r': - st = 604 - default: - return length - } - - case 604: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 605 - default: - return length - } - - case 606: - switch byteutil.ByteToLower(b) { - case 'o': - st = 607 - default: - return length - } - - case 607: - switch byteutil.ByteToLower(b) { - case 'c': - st = 608 - default: - return length - } - - case 608: - switch byteutil.ByteToLower(b) { - case 'r': - st = 609 - default: - return length - } - - case 609: - switch byteutil.ByteToLower(b) { - case 'a': - st = 610 - default: - return length - } - - case 610: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 611 - default: - return length - } - - case 612: - switch byteutil.ByteToLower(b) { - case 't': - st = 613 - default: - return length - } - - case 613: - switch byteutil.ByteToLower(b) { - case 'a': - st = 614 - case 'i': - st = 616 - default: - return length - } - - case 614: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 615 - default: - return length - } - - case 616: - switch byteutil.ByteToLower(b) { - case 's': - st = 617 - default: - return length - } - - case 617: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 618 - default: - return length - } - - case 619: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 620 - default: - return length - } - - case 620: - switch byteutil.ByteToLower(b) { - case 'g': - st = 621 - default: - return length - } - - case 621: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 622 - default: - return length - } - - case 624: - switch byteutil.ByteToLower(b) { - case 'a': - st = 625 - case 'e': - st = 631 - case 'g': - st = 633 - case 'r': - st = 638 - case 's': - st = 645 - default: - return length - } - - case 625: - switch byteutil.ByteToLower(b) { - case 'm': - st = 626 - default: - return length - } - - case 626: - switch byteutil.ByteToLower(b) { - case 'o': - st = 627 - default: - return length - } - - case 627: - switch byteutil.ByteToLower(b) { - case 'n': - st = 628 - default: - return length - } - - case 628: - switch byteutil.ByteToLower(b) { - case 'd': - st = 629 - default: - return length - } - - case 629: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 630 - default: - return length - } - - case 631: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 632 - default: - return length - } - - case 633: - switch byteutil.ByteToLower(b) { - case 'i': - st = 634 - default: - return length - } - - case 634: - switch byteutil.ByteToLower(b) { - case 't': - st = 635 - default: - return length - } - - case 635: - switch byteutil.ByteToLower(b) { - case 'a': - st = 636 - default: - return length - } - - case 636: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 637 - default: - return length - } - - case 638: - switch byteutil.ByteToLower(b) { - case 'e': - st = 639 - default: - return length - } - - case 639: - switch byteutil.ByteToLower(b) { - case 'c': - st = 640 - default: - return length - } - - case 640: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 641 - default: - return length - } - - case 641: - switch byteutil.ByteToLower(b) { - case 'o': - st = 642 - default: - return length - } - - case 642: - switch byteutil.ByteToLower(b) { - case 'r': - st = 643 - default: - return length - } - - case 643: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 644 - default: - return length - } - - case 645: - switch byteutil.ByteToLower(b) { - case 'c': - st = 646 - default: - return length - } - - case 646: - switch byteutil.ByteToLower(b) { - case 'o': - st = 647 - default: - return length - } - - case 647: - switch byteutil.ByteToLower(b) { - case 'u': - st = 648 - default: - return length - } - - case 648: - switch byteutil.ByteToLower(b) { - case 'n': - st = 649 - default: - return length - } - - case 649: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 650 - default: - return length - } - - case 654: - switch byteutil.ByteToLower(b) { - case 'p': - length = i + 1 - st = 655 - default: - return length - } - - case 656: - switch byteutil.ByteToLower(b) { - case 'c': - st = 657 - case 'g': - length = i + 1 - st = 659 - case 'h': - st = 660 - case 'm': - st = 662 - case 'o': - st = 667 - case 'w': - st = 671 - default: - return length - } - - case 657: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 658 - default: - return length - } - - case 660: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 661 - default: - return length - } - - case 662: - switch byteutil.ByteToLower(b) { - case 'a': - st = 663 - default: - return length - } - - case 663: - switch byteutil.ByteToLower(b) { - case 'i': - st = 664 - default: - return length - } - - case 664: - switch byteutil.ByteToLower(b) { - case 'n': - st = 665 - default: - return length - } - - case 665: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 666 - default: - return length - } - - case 667: - switch byteutil.ByteToLower(b) { - case 's': - st = 668 - default: - return length - } - - case 668: - switch byteutil.ByteToLower(b) { - case 'a': - st = 669 - default: - return length - } - - case 669: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 670 - default: - return length - } - - case 671: - switch byteutil.ByteToLower(b) { - case 'n': - st = 672 - default: - return length - } - - case 672: - switch byteutil.ByteToLower(b) { - case 'l': - st = 673 - default: - return length - } - - case 673: - switch byteutil.ByteToLower(b) { - case 'o': - st = 674 - default: - return length - } - - case 674: - switch byteutil.ByteToLower(b) { - case 'a': - st = 675 - default: - return length - } - - case 675: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 676 - default: - return length - } - - case 677: - switch byteutil.ByteToLower(b) { - case 'r': - st = 678 - default: - return length - } - - case 678: - switch byteutil.ByteToLower(b) { - case 'b': - st = 679 - default: - return length - } - - case 679: - switch byteutil.ByteToLower(b) { - case 'a': - st = 680 - default: - return length - } - - case 680: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 681 - default: - return length - } - - case 682: - switch byteutil.ByteToLower(b) { - case 'a': - st = 683 - default: - return length - } - - case 683: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 684 - default: - return length - } - - case 686: - switch byteutil.ByteToLower(b) { - case 'a': - st = 687 - case 'c': - length = i + 1 - st = 692 - case 'd': - st = 693 - case 'e': - length = i + 1 - st = 701 - case 'g': - length = i + 1 - st = 702 - case 'm': - st = 703 - case 'n': - st = 711 - case 'p': - st = 734 - case 'q': - st = 738 - case 'r': - length = i + 1 - st = 746 - case 's': - length = i + 1 - st = 749 - case 't': - length = i + 1 - st = 755 - case 'u': - length = i + 1 - st = 756 - case 'v': - st = 766 - case 'x': - st = 776 - default: - return length - } - - case 687: - switch byteutil.ByteToLower(b) { - case 'r': - st = 688 - case 't': - length = i + 1 - st = 691 - default: - return length - } - - case 688: - switch byteutil.ByteToLower(b) { - case 't': - st = 689 - default: - return length - } - - case 689: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 690 - default: - return length - } - - case 693: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 694 - default: - return length - } - - case 694: - switch byteutil.ByteToLower(b) { - case 'c': - st = 695 - default: - return length - } - - case 695: - switch byteutil.ByteToLower(b) { - case 'a': - st = 696 - default: - return length - } - - case 696: - switch byteutil.ByteToLower(b) { - case 't': - st = 697 - default: - return length - } - - case 697: - switch byteutil.ByteToLower(b) { - case 'i': - st = 698 - default: - return length - } - - case 698: - switch byteutil.ByteToLower(b) { - case 'o': - st = 699 - default: - return length - } - - case 699: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 700 - default: - return length - } - - case 703: - switch byteutil.ByteToLower(b) { - case 'a': - st = 704 - case 'e': - st = 707 - default: - return length - } - - case 704: - switch byteutil.ByteToLower(b) { - case 'i': - st = 705 - default: - return length - } - - case 705: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 706 - default: - return length - } - - case 707: - switch byteutil.ByteToLower(b) { - case 'r': - st = 708 - default: - return length - } - - case 708: - switch byteutil.ByteToLower(b) { - case 'c': - st = 709 - default: - return length - } - - case 709: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 710 - default: - return length - } - - case 711: - switch byteutil.ByteToLower(b) { - case 'e': - st = 712 - case 'g': - st = 716 - case 't': - st = 725 - default: - return length - } - - case 712: - switch byteutil.ByteToLower(b) { - case 'r': - st = 713 - default: - return length - } - - case 713: - switch byteutil.ByteToLower(b) { - case 'g': - st = 714 - default: - return length - } - - case 714: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 715 - default: - return length - } - - case 716: - switch byteutil.ByteToLower(b) { - case 'i': - st = 717 - default: - return length - } - - case 717: - switch byteutil.ByteToLower(b) { - case 'n': - st = 718 - default: - return length - } - - case 718: - switch byteutil.ByteToLower(b) { - case 'e': - st = 719 - default: - return length - } - - case 719: - switch byteutil.ByteToLower(b) { - case 'e': - st = 720 - default: - return length - } - - case 720: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 721 - default: - return length - } - - case 721: - switch byteutil.ByteToLower(b) { - case 'i': - st = 722 - default: - return length - } - - case 722: - switch byteutil.ByteToLower(b) { - case 'n': - st = 723 - default: - return length - } - - case 723: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 724 - default: - return length - } - - case 725: - switch byteutil.ByteToLower(b) { - case 'e': - st = 726 - default: - return length - } - - case 726: - switch byteutil.ByteToLower(b) { - case 'r': - st = 727 - default: - return length - } - - case 727: - switch byteutil.ByteToLower(b) { - case 'p': - st = 728 - default: - return length - } - - case 728: - switch byteutil.ByteToLower(b) { - case 'r': - st = 729 - default: - return length - } - - case 729: - switch byteutil.ByteToLower(b) { - case 'i': - st = 730 - default: - return length - } - - case 730: - switch byteutil.ByteToLower(b) { - case 's': - st = 731 - default: - return length - } - - case 731: - switch byteutil.ByteToLower(b) { - case 'e': - st = 732 - default: - return length - } - - case 732: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 733 - default: - return length - } - - case 734: - switch byteutil.ByteToLower(b) { - case 's': - st = 735 - default: - return length - } - - case 735: - switch byteutil.ByteToLower(b) { - case 'o': - st = 736 - default: - return length - } - - case 736: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 737 - default: - return length - } - - case 738: - switch byteutil.ByteToLower(b) { - case 'u': - st = 739 - default: - return length - } - - case 739: - switch byteutil.ByteToLower(b) { - case 'i': - st = 740 - default: - return length - } - - case 740: - switch byteutil.ByteToLower(b) { - case 'p': - st = 741 - default: - return length - } - - case 741: - switch byteutil.ByteToLower(b) { - case 'm': - st = 742 - default: - return length - } - - case 742: - switch byteutil.ByteToLower(b) { - case 'e': - st = 743 - default: - return length - } - - case 743: - switch byteutil.ByteToLower(b) { - case 'n': - st = 744 - default: - return length - } - - case 744: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 745 - default: - return length - } - - case 746: - switch byteutil.ByteToLower(b) { - case 'n': - st = 747 - default: - return length - } - - case 747: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 748 - default: - return length - } - - case 749: - switch byteutil.ByteToLower(b) { - case 'q': - length = i + 1 - st = 750 - case 't': - st = 751 - default: - return length - } - - case 751: - switch byteutil.ByteToLower(b) { - case 'a': - st = 752 - default: - return length - } - - case 752: - switch byteutil.ByteToLower(b) { - case 't': - st = 753 - default: - return length - } - - case 753: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 754 - default: - return length - } - - case 756: - switch byteutil.ByteToLower(b) { - case 'r': - st = 757 - case 's': - length = i + 1 - st = 765 - default: - return length - } - - case 757: - switch byteutil.ByteToLower(b) { - case 'o': - st = 758 - default: - return length - } - - case 758: - switch byteutil.ByteToLower(b) { - case 'v': - st = 759 - default: - return length - } - - case 759: - switch byteutil.ByteToLower(b) { - case 'i': - st = 760 - default: - return length - } - - case 760: - switch byteutil.ByteToLower(b) { - case 's': - st = 761 - default: - return length - } - - case 761: - switch byteutil.ByteToLower(b) { - case 'i': - st = 762 - default: - return length - } - - case 762: - switch byteutil.ByteToLower(b) { - case 'o': - st = 763 - default: - return length - } - - case 763: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 764 - default: - return length - } - - case 766: - switch byteutil.ByteToLower(b) { - case 'e': - st = 767 - default: - return length - } - - case 767: - switch byteutil.ByteToLower(b) { - case 'n': - st = 768 - case 'r': - st = 771 - default: - return length - } - - case 768: - switch byteutil.ByteToLower(b) { - case 't': - st = 769 - default: - return length - } - - case 769: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 770 - default: - return length - } - - case 771: - switch byteutil.ByteToLower(b) { - case 'b': - st = 772 - default: - return length - } - - case 772: - switch byteutil.ByteToLower(b) { - case 'a': - st = 773 - default: - return length - } - - case 773: - switch byteutil.ByteToLower(b) { - case 'n': - st = 774 - default: - return length - } - - case 774: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 775 - default: - return length - } - - case 776: - switch byteutil.ByteToLower(b) { - case 'c': - st = 777 - case 'i': - st = 783 - case 'p': - st = 785 - default: - return length - } - - case 777: - switch byteutil.ByteToLower(b) { - case 'h': - st = 778 - default: - return length - } - - case 778: - switch byteutil.ByteToLower(b) { - case 'a': - st = 779 - default: - return length - } - - case 779: - switch byteutil.ByteToLower(b) { - case 'n': - st = 780 - default: - return length - } - - case 780: - switch byteutil.ByteToLower(b) { - case 'g': - st = 781 - default: - return length - } - - case 781: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 782 - default: - return length - } - - case 783: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 784 - default: - return length - } - - case 785: - switch byteutil.ByteToLower(b) { - case 'e': - st = 786 - case 'o': - st = 789 - case 'r': - st = 793 - default: - return length - } - - case 786: - switch byteutil.ByteToLower(b) { - case 'r': - st = 787 - default: - return length - } - - case 787: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 788 - default: - return length - } - - case 789: - switch byteutil.ByteToLower(b) { - case 's': - st = 790 - default: - return length - } - - case 790: - switch byteutil.ByteToLower(b) { - case 'e': - st = 791 - default: - return length - } - - case 791: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 792 - default: - return length - } - - case 793: - switch byteutil.ByteToLower(b) { - case 'e': - st = 794 - default: - return length - } - - case 794: - switch byteutil.ByteToLower(b) { - case 's': - st = 795 - default: - return length - } - - case 795: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 796 - default: - return length - } - - case 797: - switch byteutil.ByteToLower(b) { - case 'a': - st = 798 - case 'e': - st = 812 - case 'i': - length = i + 1 - st = 819 - case 'j': - length = i + 1 - st = 846 - case 'k': - length = i + 1 - st = 847 - case 'l': - st = 848 - case 'm': - length = i + 1 - st = 870 - case 'o': - length = i + 1 - st = 871 - case 'r': - length = i + 1 - st = 893 - case 'u': - st = 900 - case 'y': - st = 914 - default: - return length - } - - case 798: - switch byteutil.ByteToLower(b) { - case 'i': - st = 799 - case 'n': - length = i + 1 - st = 803 - case 'r': - st = 805 - case 's': - st = 807 - default: - return length - } - - case 799: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 800 - case 't': - st = 801 - default: - return length - } - - case 801: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 802 - default: - return length - } - - case 803: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 804 - default: - return length - } - - case 805: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 806 - default: - return length - } - - case 807: - switch byteutil.ByteToLower(b) { - case 'h': - st = 808 - default: - return length - } - - case 808: - switch byteutil.ByteToLower(b) { - case 'i': - st = 809 - default: - return length - } - - case 809: - switch byteutil.ByteToLower(b) { - case 'o': - st = 810 - default: - return length - } - - case 810: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 811 - default: - return length - } - - case 812: - switch byteutil.ByteToLower(b) { - case 'e': - st = 813 - default: - return length - } - - case 813: - switch byteutil.ByteToLower(b) { - case 'd': - st = 814 - default: - return length - } - - case 814: - switch byteutil.ByteToLower(b) { - case 'b': - st = 815 - default: - return length - } - - case 815: - switch byteutil.ByteToLower(b) { - case 'a': - st = 816 - default: - return length - } - - case 816: - switch byteutil.ByteToLower(b) { - case 'c': - st = 817 - default: - return length - } - - case 817: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 818 - default: - return length - } - - case 819: - switch byteutil.ByteToLower(b) { - case 'l': - st = 820 - case 'n': - st = 822 - case 'r': - st = 830 - case 's': - st = 836 - case 't': - length = i + 1 - st = 841 - default: - return length - } - - case 820: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 821 - default: - return length - } - - case 822: - switch byteutil.ByteToLower(b) { - case 'a': - st = 823 - default: - return length - } - - case 823: - switch byteutil.ByteToLower(b) { - case 'n': - st = 824 - default: - return length - } - - case 824: - switch byteutil.ByteToLower(b) { - case 'c': - st = 825 - default: - return length - } - - case 825: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 826 - case 'i': - st = 827 - default: - return length - } - - case 827: - switch byteutil.ByteToLower(b) { - case 'a': - st = 828 - default: - return length - } - - case 828: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 829 - default: - return length - } - - case 830: - switch byteutil.ByteToLower(b) { - case 'm': - st = 831 - default: - return length - } - - case 831: - switch byteutil.ByteToLower(b) { - case 'd': - st = 832 - default: - return length - } - - case 832: - switch byteutil.ByteToLower(b) { - case 'a': - st = 833 - default: - return length - } - - case 833: - switch byteutil.ByteToLower(b) { - case 'l': - st = 834 - default: - return length - } - - case 834: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 835 - default: - return length - } - - case 836: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 837 - default: - return length - } - - case 837: - switch byteutil.ByteToLower(b) { - case 'i': - st = 838 - default: - return length - } - - case 838: - switch byteutil.ByteToLower(b) { - case 'n': - st = 839 - default: - return length - } - - case 839: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 840 - default: - return length - } - - case 841: - switch byteutil.ByteToLower(b) { - case 'n': - st = 842 - default: - return length - } - - case 842: - switch byteutil.ByteToLower(b) { - case 'e': - st = 843 - default: - return length - } - - case 843: - switch byteutil.ByteToLower(b) { - case 's': - st = 844 - default: - return length - } - - case 844: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 845 - default: - return length - } - - case 848: - switch byteutil.ByteToLower(b) { - case 'i': - st = 849 - case 'o': - st = 854 - case 's': - st = 863 - case 'y': - length = i + 1 - st = 869 - default: - return length - } - - case 849: - switch byteutil.ByteToLower(b) { - case 'g': - st = 850 - default: - return length - } - - case 850: - switch byteutil.ByteToLower(b) { - case 'h': - st = 851 - default: - return length - } - - case 851: - switch byteutil.ByteToLower(b) { - case 't': - st = 852 - default: - return length - } - - case 852: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 853 - default: - return length - } - - case 854: - switch byteutil.ByteToLower(b) { - case 'r': - st = 855 - case 'w': - st = 859 - default: - return length - } - - case 855: - switch byteutil.ByteToLower(b) { - case 'i': - st = 856 - default: - return length - } - - case 856: - switch byteutil.ByteToLower(b) { - case 's': - st = 857 - default: - return length - } - - case 857: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 858 - default: - return length - } - - case 859: - switch byteutil.ByteToLower(b) { - case 'e': - st = 860 - default: - return length - } - - case 860: - switch byteutil.ByteToLower(b) { - case 'r': - st = 861 - default: - return length - } - - case 861: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 862 - default: - return length - } - - case 863: - switch byteutil.ByteToLower(b) { - case 'm': - st = 864 - default: - return length - } - - case 864: - switch byteutil.ByteToLower(b) { - case 'i': - st = 865 - default: - return length - } - - case 865: - switch byteutil.ByteToLower(b) { - case 'd': - st = 866 - default: - return length - } - - case 866: - switch byteutil.ByteToLower(b) { - case 't': - st = 867 - default: - return length - } - - case 867: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 868 - default: - return length - } - - case 871: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 872 - case 'r': - st = 878 - case 'u': - st = 885 - default: - return length - } - - case 872: - switch byteutil.ByteToLower(b) { - case 't': - st = 873 - default: - return length - } - - case 873: - switch byteutil.ByteToLower(b) { - case 'b': - st = 874 - default: - return length - } - - case 874: - switch byteutil.ByteToLower(b) { - case 'a': - st = 875 - default: - return length - } - - case 875: - switch byteutil.ByteToLower(b) { - case 'l': - st = 876 - default: - return length - } - - case 876: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 877 - default: - return length - } - - case 878: - switch byteutil.ByteToLower(b) { - case 'e': - st = 879 - case 's': - st = 881 - default: - return length - } - - case 879: - switch byteutil.ByteToLower(b) { - case 'x': - length = i + 1 - st = 880 - default: - return length - } - - case 881: - switch byteutil.ByteToLower(b) { - case 'a': - st = 882 - default: - return length - } - - case 882: - switch byteutil.ByteToLower(b) { - case 'l': - st = 883 - default: - return length - } - - case 883: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 884 - default: - return length - } - - case 885: - switch byteutil.ByteToLower(b) { - case 'n': - st = 886 - default: - return length - } - - case 886: - switch byteutil.ByteToLower(b) { - case 'd': - st = 887 - default: - return length - } - - case 887: - switch byteutil.ByteToLower(b) { - case 'a': - st = 888 - default: - return length - } - - case 888: - switch byteutil.ByteToLower(b) { - case 't': - st = 889 - default: - return length - } - - case 889: - switch byteutil.ByteToLower(b) { - case 'i': - st = 890 - default: - return length - } - - case 890: - switch byteutil.ByteToLower(b) { - case 'o': - st = 891 - default: - return length - } - - case 891: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 892 - default: - return length - } - - case 893: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 894 - case 'o': - st = 895 - default: - return length - } - - case 895: - switch byteutil.ByteToLower(b) { - case 'g': - st = 896 - default: - return length - } - - case 896: - switch byteutil.ByteToLower(b) { - case 'a': - st = 897 - default: - return length - } - - case 897: - switch byteutil.ByteToLower(b) { - case 'n': - st = 898 - default: - return length - } - - case 898: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 899 - default: - return length - } - - case 900: - switch byteutil.ByteToLower(b) { - case 'n': - st = 901 - case 'r': - st = 903 - case 't': - st = 910 - default: - return length - } - - case 901: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 902 - default: - return length - } - - case 903: - switch byteutil.ByteToLower(b) { - case 'n': - st = 904 - default: - return length - } - - case 904: - switch byteutil.ByteToLower(b) { - case 'i': - st = 905 - default: - return length - } - - case 905: - switch byteutil.ByteToLower(b) { - case 't': - st = 906 - default: - return length - } - - case 906: - switch byteutil.ByteToLower(b) { - case 'u': - st = 907 - default: - return length - } - - case 907: - switch byteutil.ByteToLower(b) { - case 'r': - st = 908 - default: - return length - } - - case 908: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 909 - default: - return length - } - - case 910: - switch byteutil.ByteToLower(b) { - case 'b': - st = 911 - default: - return length - } - - case 911: - switch byteutil.ByteToLower(b) { - case 'o': - st = 912 - default: - return length - } - - case 912: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 913 - default: - return length - } - - case 914: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 915 - default: - return length - } - - case 916: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 917 - case 'b': - length = i + 1 - st = 927 - case 'd': - length = i + 1 - st = 930 - case 'e': - length = i + 1 - st = 932 - case 'f': - length = i + 1 - st = 935 - case 'g': - length = i + 1 - st = 936 - case 'h': - length = i + 1 - st = 939 - case 'i': - length = i + 1 - st = 940 - case 'l': - length = i + 1 - st = 947 - case 'm': - length = i + 1 - st = 957 - case 'n': - length = i + 1 - st = 963 - case 'o': - st = 965 - case 'p': - length = i + 1 - st = 980 - case 'q': - length = i + 1 - st = 981 - case 'r': - length = i + 1 - st = 982 - case 's': - length = i + 1 - st = 998 - case 't': - length = i + 1 - st = 999 - case 'u': - length = i + 1 - st = 1000 - case 'w': - length = i + 1 - st = 1012 - case 'y': - length = i + 1 - st = 1013 - default: - return length - } - - case 917: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 918 - case 'r': - st = 923 - default: - return length - } - - case 918: - switch byteutil.ByteToLower(b) { - case 'l': - st = 919 - default: - return length - } - - case 919: - switch byteutil.ByteToLower(b) { - case 'e': - st = 920 - default: - return length - } - - case 920: - switch byteutil.ByteToLower(b) { - case 'r': - st = 921 - default: - return length - } - - case 921: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 922 - default: - return length - } - - case 923: - switch byteutil.ByteToLower(b) { - case 'd': - st = 924 - default: - return length - } - - case 924: - switch byteutil.ByteToLower(b) { - case 'e': - st = 925 - default: - return length - } - - case 925: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 926 - default: - return length - } - - case 927: - switch byteutil.ByteToLower(b) { - case 'i': - st = 928 - default: - return length - } - - case 928: - switch byteutil.ByteToLower(b) { - case 'z': - length = i + 1 - st = 929 - default: - return length - } - - case 930: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 931 - default: - return length - } - - case 932: - switch byteutil.ByteToLower(b) { - case 'n': - st = 933 - default: - return length - } - - case 933: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 934 - default: - return length - } - - case 936: - switch byteutil.ByteToLower(b) { - case 'e': - st = 937 - default: - return length - } - - case 937: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 938 - default: - return length - } - - case 940: - switch byteutil.ByteToLower(b) { - case 'f': - st = 941 - case 'v': - st = 944 - default: - return length - } - - case 941: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 942 - default: - return length - } - - case 942: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 943 - default: - return length - } - - case 944: - switch byteutil.ByteToLower(b) { - case 'e': - st = 945 - default: - return length - } - - case 945: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 946 - default: - return length - } - - case 947: - switch byteutil.ByteToLower(b) { - case 'a': - st = 948 - case 'e': - length = i + 1 - st = 951 - case 'o': - st = 952 - default: - return length - } - - case 948: - switch byteutil.ByteToLower(b) { - case 's': - st = 949 - default: - return length - } - - case 949: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 950 - default: - return length - } - - case 952: - switch byteutil.ByteToLower(b) { - case 'b': - st = 953 - default: - return length - } - - case 953: - switch byteutil.ByteToLower(b) { - case 'a': - st = 954 - case 'o': - length = i + 1 - st = 956 - default: - return length - } - - case 954: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 955 - default: - return length - } - - case 957: - switch byteutil.ByteToLower(b) { - case 'a': - st = 958 - case 'o': - length = i + 1 - st = 961 - case 'x': - length = i + 1 - st = 962 - default: - return length - } - - case 958: - switch byteutil.ByteToLower(b) { - case 'i': - st = 959 - default: - return length - } - - case 959: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 960 - default: - return length - } - - case 963: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 964 - default: - return length - } - - case 965: - switch byteutil.ByteToLower(b) { - case 'l': - st = 966 - case 'o': - length = i + 1 - st = 974 - case 'p': - length = i + 1 - st = 978 - case 'v': - length = i + 1 - st = 979 - default: - return length - } - - case 966: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 967 - case 'f': - length = i + 1 - st = 973 - default: - return length - } - - case 967: - switch byteutil.ByteToLower(b) { - case 'p': - st = 968 - default: - return length - } - - case 968: - switch byteutil.ByteToLower(b) { - case 'o': - st = 969 - default: - return length - } - - case 969: - switch byteutil.ByteToLower(b) { - case 'i': - st = 970 - default: - return length - } - - case 970: - switch byteutil.ByteToLower(b) { - case 'n': - st = 971 - default: - return length - } - - case 971: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 972 - default: - return length - } - - case 974: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 975 - default: - return length - } - - case 975: - switch byteutil.ByteToLower(b) { - case 'l': - st = 976 - default: - return length - } - - case 976: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 977 - default: - return length - } - - case 982: - switch byteutil.ByteToLower(b) { - case 'a': - st = 983 - case 'e': - st = 992 - case 'i': - st = 995 - default: - return length - } - - case 983: - switch byteutil.ByteToLower(b) { - case 'p': - st = 984 - case 't': - st = 989 - default: - return length - } - - case 984: - switch byteutil.ByteToLower(b) { - case 'h': - st = 985 - default: - return length - } - - case 985: - switch byteutil.ByteToLower(b) { - case 'i': - st = 986 - default: - return length - } - - case 986: - switch byteutil.ByteToLower(b) { - case 'c': - st = 987 - default: - return length - } - - case 987: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 988 - default: - return length - } - - case 989: - switch byteutil.ByteToLower(b) { - case 'i': - st = 990 - default: - return length - } - - case 990: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 991 - default: - return length - } - - case 992: - switch byteutil.ByteToLower(b) { - case 'e': - st = 993 - default: - return length - } - - case 993: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 994 - default: - return length - } - - case 995: - switch byteutil.ByteToLower(b) { - case 'p': - st = 996 - default: - return length - } - - case 996: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 997 - default: - return length - } - - case 1000: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1001 - case 'i': - st = 1003 - case 'r': - st = 1010 - default: - return length - } - - case 1001: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1002 - default: - return length - } - - case 1003: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1004 - case 't': - st = 1006 - default: - return length - } - - case 1004: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1005 - default: - return length - } - - case 1006: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1007 - default: - return length - } - - case 1007: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1008 - default: - return length - } - - case 1008: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1009 - default: - return length - } - - case 1010: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 1011 - default: - return length - } - - case 1014: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1015 - case 'e': - st = 1028 - case 'i': - st = 1044 - case 'k': - length = i + 1 - st = 1055 - case 'm': - length = i + 1 - st = 1056 - case 'n': - length = i + 1 - st = 1057 - case 'o': - st = 1058 - case 'r': - length = i + 1 - st = 1096 - case 't': - length = i + 1 - st = 1097 - case 'u': - length = i + 1 - st = 1098 - default: - return length - } - - case 1015: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1016 - case 'n': - st = 1021 - case 'u': - st = 1026 - default: - return length - } - - case 1016: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1017 - default: - return length - } - - case 1017: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1018 - default: - return length - } - - case 1018: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1019 - default: - return length - } - - case 1019: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1020 - default: - return length - } - - case 1021: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1022 - default: - return length - } - - case 1022: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1023 - default: - return length - } - - case 1023: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1024 - default: - return length - } - - case 1024: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1025 - default: - return length - } - - case 1026: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1027 - default: - return length - } - - case 1028: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1029 - case 'l': - st = 1037 - case 'r': - st = 1039 - default: - return length - } - - case 1029: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1030 - default: - return length - } - - case 1030: - switch byteutil.ByteToLower(b) { - case 't': - st = 1031 - default: - return length - } - - case 1031: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1032 - default: - return length - } - - case 1032: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1033 - default: - return length - } - - case 1033: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1034 - default: - return length - } - - case 1034: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1035 - default: - return length - } - - case 1035: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1036 - default: - return length - } - - case 1037: - switch byteutil.ByteToLower(b) { - case 'p': - length = i + 1 - st = 1038 - default: - return length - } - - case 1039: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1040 - case 'm': - st = 1041 - default: - return length - } - - case 1041: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1042 - default: - return length - } - - case 1042: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1043 - default: - return length - } - - case 1044: - switch byteutil.ByteToLower(b) { - case 'p': - st = 1045 - case 't': - st = 1049 - case 'v': - length = i + 1 - st = 1054 - default: - return length - } - - case 1045: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1046 - default: - return length - } - - case 1046: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1047 - default: - return length - } - - case 1047: - switch byteutil.ByteToLower(b) { - case 'p': - length = i + 1 - st = 1048 - default: - return length - } - - case 1049: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1050 - default: - return length - } - - case 1050: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1051 - default: - return length - } - - case 1051: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1052 - default: - return length - } - - case 1052: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1053 - default: - return length - } - - case 1058: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1059 - case 'l': - st = 1063 - case 'm': - st = 1073 - case 'n': - st = 1081 - case 'r': - st = 1084 - case 's': - st = 1087 - case 'u': - st = 1092 - case 'w': - length = i + 1 - st = 1095 - default: - return length - } - - case 1059: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1060 - default: - return length - } - - case 1060: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1061 - default: - return length - } - - case 1061: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1062 - default: - return length - } - - case 1063: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1064 - case 'i': - st = 1069 - default: - return length - } - - case 1064: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1065 - default: - return length - } - - case 1065: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1066 - default: - return length - } - - case 1066: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1067 - default: - return length - } - - case 1067: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1068 - default: - return length - } - - case 1069: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1070 - default: - return length - } - - case 1070: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1071 - default: - return length - } - - case 1071: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1072 - default: - return length - } - - case 1073: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1074 - default: - return length - } - - case 1074: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1075 - case 's': - length = i + 1 - st = 1080 - default: - return length - } - - case 1075: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1076 - default: - return length - } - - case 1076: - switch byteutil.ByteToLower(b) { - case 'p': - st = 1077 - default: - return length - } - - case 1077: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1078 - default: - return length - } - - case 1078: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1079 - default: - return length - } - - case 1081: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1082 - default: - return length - } - - case 1082: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1083 - default: - return length - } - - case 1084: - switch byteutil.ByteToLower(b) { - case 's': - st = 1085 - default: - return length - } - - case 1085: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1086 - default: - return length - } - - case 1087: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1088 - default: - return length - } - - case 1088: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1089 - default: - return length - } - - case 1089: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1090 - default: - return length - } - - case 1090: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1091 - default: - return length - } - - case 1092: - switch byteutil.ByteToLower(b) { - case 's': - st = 1093 - default: - return length - } - - case 1093: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1094 - default: - return length - } - - case 1099: - switch byteutil.ByteToLower(b) { - case '2': - st = 1100 - case 'b': - st = 1102 - case 'c': - st = 1104 - case 'd': - length = i + 1 - st = 1108 - case 'e': - length = i + 1 - st = 1109 - case 'f': - st = 1110 - case 'l': - length = i + 1 - st = 1112 - case 'm': - length = i + 1 - st = 1113 - case 'n': - length = i + 1 - st = 1122 - case 'o': - length = i + 1 - st = 1170 - case 'q': - length = i + 1 - st = 1171 - case 'r': - length = i + 1 - st = 1172 - case 's': - length = i + 1 - st = 1176 - case 't': - length = i + 1 - st = 1177 - case 'w': - st = 1178 - default: - return length - } - - case 1100: - switch byteutil.ByteToLower(b) { - case 'p': - length = i + 1 - st = 1101 - default: - return length - } - - case 1102: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 1103 - default: - return length - } - - case 1104: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1105 - case 'u': - length = i + 1 - st = 1107 - default: - return length - } - - case 1105: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 1106 - default: - return length - } - - case 1110: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 1111 - default: - return length - } - - case 1113: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1114 - default: - return length - } - - case 1114: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1115 - default: - return length - } - - case 1115: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1116 - default: - return length - } - - case 1116: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1117 - default: - return length - } - - case 1117: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1118 - default: - return length - } - - case 1118: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1119 - default: - return length - } - - case 1119: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1120 - default: - return length - } - - case 1120: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1121 - default: - return length - } - - case 1122: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1123 - case 'f': - st = 1131 - case 'g': - length = i + 1 - st = 1138 - case 'k': - length = i + 1 - st = 1139 - case 's': - st = 1140 - case 't': - length = i + 1 - st = 1150 - case 'v': - st = 1161 - default: - return length - } - - case 1123: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1124 - default: - return length - } - - case 1124: - switch byteutil.ByteToLower(b) { - case 's': - st = 1125 - default: - return length - } - - case 1125: - switch byteutil.ByteToLower(b) { - case 't': - st = 1126 - default: - return length - } - - case 1126: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1127 - default: - return length - } - - case 1127: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1128 - default: - return length - } - - case 1128: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1129 - default: - return length - } - - case 1129: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1130 - default: - return length - } - - case 1131: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1132 - case 'o': - length = i + 1 - st = 1137 - default: - return length - } - - case 1132: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1133 - default: - return length - } - - case 1133: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1134 - default: - return length - } - - case 1134: - switch byteutil.ByteToLower(b) { - case 't': - st = 1135 - default: - return length - } - - case 1135: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1136 - default: - return length - } - - case 1140: - switch byteutil.ByteToLower(b) { - case 't': - st = 1141 - case 'u': - st = 1147 - default: - return length - } - - case 1141: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1142 - default: - return length - } - - case 1142: - switch byteutil.ByteToLower(b) { - case 't': - st = 1143 - default: - return length - } - - case 1143: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1144 - default: - return length - } - - case 1144: - switch byteutil.ByteToLower(b) { - case 't': - st = 1145 - default: - return length - } - - case 1145: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1146 - default: - return length - } - - case 1147: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1148 - default: - return length - } - - case 1148: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1149 - default: - return length - } - - case 1150: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1151 - default: - return length - } - - case 1151: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1152 - default: - return length - } - - case 1152: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1153 - default: - return length - } - - case 1153: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1154 - default: - return length - } - - case 1154: - switch byteutil.ByteToLower(b) { - case 't': - st = 1155 - default: - return length - } - - case 1155: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1156 - default: - return length - } - - case 1156: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1157 - default: - return length - } - - case 1157: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1158 - default: - return length - } - - case 1158: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1159 - default: - return length - } - - case 1159: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1160 - default: - return length - } - - case 1161: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1162 - default: - return length - } - - case 1162: - switch byteutil.ByteToLower(b) { - case 's': - st = 1163 - default: - return length - } - - case 1163: - switch byteutil.ByteToLower(b) { - case 't': - st = 1164 - default: - return length - } - - case 1164: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1165 - default: - return length - } - - case 1165: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1166 - default: - return length - } - - case 1166: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1167 - default: - return length - } - - case 1167: - switch byteutil.ByteToLower(b) { - case 't': - st = 1168 - default: - return length - } - - case 1168: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1169 - default: - return length - } - - case 1172: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1173 - default: - return length - } - - case 1173: - switch byteutil.ByteToLower(b) { - case 's': - st = 1174 - default: - return length - } - - case 1174: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 1175 - default: - return length - } - - case 1178: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 1179 - default: - return length - } - - case 1180: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1181 - case 'c': - st = 1184 - case 'e': - length = i + 1 - st = 1186 - case 'l': - st = 1195 - case 'm': - length = i + 1 - st = 1197 - case 'o': - length = i + 1 - st = 1198 - case 'p': - length = i + 1 - st = 1204 - case 'u': - st = 1205 - default: - return length - } - - case 1181: - switch byteutil.ByteToLower(b) { - case 'v': - st = 1182 - default: - return length - } - - case 1182: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1183 - default: - return length - } - - case 1184: - switch byteutil.ByteToLower(b) { - case 'b': - length = i + 1 - st = 1185 - default: - return length - } - - case 1186: - switch byteutil.ByteToLower(b) { - case 't': - st = 1187 - case 'w': - st = 1190 - default: - return length - } - - case 1187: - switch byteutil.ByteToLower(b) { - case 'z': - st = 1188 - default: - return length - } - - case 1188: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1189 - default: - return length - } - - case 1190: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1191 - default: - return length - } - - case 1191: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1192 - default: - return length - } - - case 1192: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1193 - default: - return length - } - - case 1193: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1194 - default: - return length - } - - case 1195: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1196 - default: - return length - } - - case 1198: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1199 - default: - return length - } - - case 1199: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1200 - case 'u': - st = 1201 - default: - return length - } - - case 1201: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1202 - default: - return length - } - - case 1202: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1203 - default: - return length - } - - case 1205: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1206 - default: - return length - } - - case 1206: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1207 - default: - return length - } - - case 1207: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1208 - default: - return length - } - - case 1208: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1209 - default: - return length - } - - case 1210: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1211 - case 'd': - st = 1216 - case 'e': - length = i + 1 - st = 1219 - case 'g': - length = i + 1 - st = 1220 - case 'h': - length = i + 1 - st = 1221 - case 'i': - length = i + 1 - st = 1222 - case 'm': - length = i + 1 - st = 1231 - case 'n': - length = i + 1 - st = 1232 - case 'o': - st = 1233 - case 'p': - length = i + 1 - st = 1242 - case 'r': - length = i + 1 - st = 1243 - case 'w': - length = i + 1 - st = 1247 - case 'y': - length = i + 1 - st = 1248 - case 'z': - length = i + 1 - st = 1252 - default: - return length - } - - case 1211: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1212 - default: - return length - } - - case 1212: - switch byteutil.ByteToLower(b) { - case 'f': - st = 1213 - default: - return length - } - - case 1213: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1214 - default: - return length - } - - case 1214: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1215 - default: - return length - } - - case 1216: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1217 - default: - return length - } - - case 1217: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1218 - default: - return length - } - - case 1222: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 1223 - case 't': - st = 1224 - case 'w': - st = 1229 - default: - return length - } - - case 1224: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1225 - default: - return length - } - - case 1225: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1226 - default: - return length - } - - case 1226: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1227 - default: - return length - } - - case 1227: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1228 - default: - return length - } - - case 1229: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1230 - default: - return length - } - - case 1233: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1234 - case 'm': - st = 1237 - default: - return length - } - - case 1234: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1235 - default: - return length - } - - case 1235: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1236 - default: - return length - } - - case 1237: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1238 - default: - return length - } - - case 1238: - switch byteutil.ByteToLower(b) { - case 't': - st = 1239 - default: - return length - } - - case 1239: - switch byteutil.ByteToLower(b) { - case 's': - st = 1240 - default: - return length - } - - case 1240: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 1241 - default: - return length - } - - case 1243: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 1244 - case 'e': - st = 1245 - default: - return length - } - - case 1245: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 1246 - default: - return length - } - - case 1248: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1249 - default: - return length - } - - case 1249: - switch byteutil.ByteToLower(b) { - case 't': - st = 1250 - default: - return length - } - - case 1250: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1251 - default: - return length - } - - case 1253: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1254 - case 'b': - length = i + 1 - st = 1271 - case 'c': - length = i + 1 - st = 1272 - case 'd': - st = 1273 - case 'e': - st = 1275 - case 'g': - st = 1287 - case 'i': - length = i + 1 - st = 1290 - case 'k': - length = i + 1 - st = 1314 - case 'o': - st = 1315 - case 'r': - length = i + 1 - st = 1333 - case 's': - length = i + 1 - st = 1334 - case 't': - length = i + 1 - st = 1335 - case 'u': - length = i + 1 - st = 1338 - case 'v': - length = i + 1 - st = 1347 - case 'y': - length = i + 1 - st = 1348 - default: - return length - } - - case 1254: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1255 - case 'n': - st = 1260 - case 't': - length = i + 1 - st = 1262 - case 'w': - st = 1267 - default: - return length - } - - case 1255: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1256 - default: - return length - } - - case 1256: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1257 - default: - return length - } - - case 1257: - switch byteutil.ByteToLower(b) { - case 'x': - st = 1258 - default: - return length - } - - case 1258: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1259 - default: - return length - } - - case 1260: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 1261 - default: - return length - } - - case 1262: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1263 - default: - return length - } - - case 1263: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1264 - default: - return length - } - - case 1264: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1265 - default: - return length - } - - case 1265: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1266 - default: - return length - } - - case 1267: - switch byteutil.ByteToLower(b) { - case 'y': - st = 1268 - default: - return length - } - - case 1268: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1269 - default: - return length - } - - case 1269: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1270 - default: - return length - } - - case 1273: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1274 - default: - return length - } - - case 1275: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1276 - case 'c': - st = 1279 - case 'g': - st = 1284 - default: - return length - } - - case 1276: - switch byteutil.ByteToLower(b) { - case 's': - st = 1277 - default: - return length - } - - case 1277: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1278 - default: - return length - } - - case 1279: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1280 - default: - return length - } - - case 1280: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1281 - default: - return length - } - - case 1281: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1282 - default: - return length - } - - case 1282: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 1283 - default: - return length - } - - case 1284: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1285 - default: - return length - } - - case 1285: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1286 - default: - return length - } - - case 1287: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1288 - default: - return length - } - - case 1288: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1289 - default: - return length - } - - case 1290: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1291 - case 'd': - st = 1296 - case 'f': - st = 1298 - case 'g': - st = 1300 - case 'm': - st = 1306 - case 'n': - st = 1312 - default: - return length - } - - case 1291: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1292 - default: - return length - } - - case 1292: - switch byteutil.ByteToLower(b) { - case 's': - st = 1293 - default: - return length - } - - case 1293: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1294 - default: - return length - } - - case 1294: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1295 - default: - return length - } - - case 1296: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1297 - default: - return length - } - - case 1298: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1299 - default: - return length - } - - case 1300: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1301 - default: - return length - } - - case 1301: - switch byteutil.ByteToLower(b) { - case 't': - st = 1302 - default: - return length - } - - case 1302: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1303 - default: - return length - } - - case 1303: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1304 - default: - return length - } - - case 1304: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1305 - default: - return length - } - - case 1306: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1307 - case 'o': - length = i + 1 - st = 1311 - default: - return length - } - - case 1307: - switch byteutil.ByteToLower(b) { - case 't': - st = 1308 - default: - return length - } - - case 1308: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1309 - default: - return length - } - - case 1309: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 1310 - default: - return length - } - - case 1312: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 1313 - default: - return length - } - - case 1315: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1316 - case 'c': - st = 1319 - case 'l': - length = i + 1 - st = 1322 - case 'n': - st = 1323 - case 't': - st = 1327 - case 'v': - st = 1331 - default: - return length - } - - case 1316: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1317 - default: - return length - } - - case 1317: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1318 - default: - return length - } - - case 1319: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1320 - default: - return length - } - - case 1320: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1321 - default: - return length - } - - case 1323: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1324 - default: - return length - } - - case 1324: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1325 - default: - return length - } - - case 1325: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1326 - default: - return length - } - - case 1327: - switch byteutil.ByteToLower(b) { - case 't': - st = 1328 - default: - return length - } - - case 1328: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1329 - case 'o': - length = i + 1 - st = 1330 - default: - return length - } - - case 1331: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1332 - default: - return length - } - - case 1335: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1336 - default: - return length - } - - case 1336: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1337 - default: - return length - } - - case 1338: - switch byteutil.ByteToLower(b) { - case 'p': - st = 1339 - case 'x': - st = 1342 - default: - return length - } - - case 1339: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1340 - default: - return length - } - - case 1340: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1341 - default: - return length - } - - case 1342: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1343 - case 'u': - st = 1344 - default: - return length - } - - case 1344: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1345 - default: - return length - } - - case 1345: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1346 - default: - return length - } - - case 1349: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1350 - case 'b': - st = 1383 - case 'c': - length = i + 1 - st = 1385 - case 'd': - length = i + 1 - st = 1386 - case 'e': - length = i + 1 - st = 1387 - case 'g': - length = i + 1 - st = 1409 - case 'h': - length = i + 1 - st = 1410 - case 'i': - st = 1411 - case 'k': - length = i + 1 - st = 1418 - case 'l': - length = i + 1 - st = 1419 - case 'm': - length = i + 1 - st = 1420 - case 'n': - length = i + 1 - st = 1422 - case 'o': - length = i + 1 - st = 1423 - case 'p': - length = i + 1 - st = 1460 - case 'q': - length = i + 1 - st = 1461 - case 'r': - length = i + 1 - st = 1462 - case 's': - length = i + 1 - st = 1463 - case 't': - length = i + 1 - st = 1464 - case 'u': - length = i + 1 - st = 1468 - case 'v': - length = i + 1 - st = 1473 - case 'w': - length = i + 1 - st = 1474 - case 'x': - length = i + 1 - st = 1475 - case 'y': - length = i + 1 - st = 1476 - case 'z': - length = i + 1 - st = 1477 - default: - return length - } - - case 1350: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1351 - case 'i': - st = 1355 - case 'n': - st = 1360 - case 'r': - st = 1370 - default: - return length - } - - case 1351: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1352 - default: - return length - } - - case 1352: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1353 - default: - return length - } - - case 1353: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 1354 - default: - return length - } - - case 1355: - switch byteutil.ByteToLower(b) { - case 'f': - length = i + 1 - st = 1356 - case 's': - st = 1357 - default: - return length - } - - case 1357: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1358 - default: - return length - } - - case 1358: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1359 - default: - return length - } - - case 1360: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1361 - case 'g': - st = 1368 - default: - return length - } - - case 1361: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1362 - default: - return length - } - - case 1362: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1363 - default: - return length - } - - case 1363: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1364 - default: - return length - } - - case 1364: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1365 - default: - return length - } - - case 1365: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1366 - default: - return length - } - - case 1366: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1367 - default: - return length - } - - case 1368: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1369 - default: - return length - } - - case 1370: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1371 - case 'r': - st = 1378 - default: - return length - } - - case 1371: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1372 - default: - return length - } - - case 1372: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1373 - default: - return length - } - - case 1373: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1374 - case 's': - length = i + 1 - st = 1377 - default: - return length - } - - case 1374: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1375 - default: - return length - } - - case 1375: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1376 - default: - return length - } - - case 1378: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1379 - default: - return length - } - - case 1379: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1380 - default: - return length - } - - case 1380: - switch byteutil.ByteToLower(b) { - case 't': - st = 1381 - default: - return length - } - - case 1381: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1382 - default: - return length - } - - case 1383: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1384 - default: - return length - } - - case 1387: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1388 - case 'e': - st = 1391 - case 'l': - st = 1393 - case 'm': - st = 1400 - case 'n': - length = i + 1 - st = 1407 - default: - return length - } - - case 1388: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1389 - default: - return length - } - - case 1389: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1390 - default: - return length - } - - case 1391: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1392 - default: - return length - } - - case 1393: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1394 - default: - return length - } - - case 1394: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1395 - default: - return length - } - - case 1395: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1396 - default: - return length - } - - case 1396: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1397 - default: - return length - } - - case 1397: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1398 - default: - return length - } - - case 1398: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1399 - default: - return length - } - - case 1400: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1401 - case 'o': - st = 1402 - default: - return length - } - - case 1402: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1403 - default: - return length - } - - case 1403: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1404 - default: - return length - } - - case 1404: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1405 - default: - return length - } - - case 1405: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1406 - default: - return length - } - - case 1407: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 1408 - default: - return length - } - - case 1411: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1412 - case 'l': - length = i + 1 - st = 1415 - case 'n': - st = 1416 - default: - return length - } - - case 1412: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1413 - default: - return length - } - - case 1413: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1414 - default: - return length - } - - case 1416: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1417 - default: - return length - } - - case 1420: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1421 - default: - return length - } - - case 1423: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1424 - case 'd': - st = 1426 - case 'e': - length = i + 1 - st = 1428 - case 'n': - st = 1429 - case 'r': - st = 1435 - case 's': - st = 1444 - case 't': - st = 1448 - case 'v': - length = i + 1 - st = 1457 - default: - return length - } - - case 1424: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1425 - default: - return length - } - - case 1426: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1427 - default: - return length - } - - case 1429: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1430 - case 'e': - st = 1433 - default: - return length - } - - case 1430: - switch byteutil.ByteToLower(b) { - case 's': - st = 1431 - default: - return length - } - - case 1431: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 1432 - default: - return length - } - - case 1433: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1434 - default: - return length - } - - case 1435: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1436 - case 't': - st = 1439 - default: - return length - } - - case 1436: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1437 - default: - return length - } - - case 1437: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1438 - default: - return length - } - - case 1439: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1440 - default: - return length - } - - case 1440: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1441 - default: - return length - } - - case 1441: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1442 - default: - return length - } - - case 1442: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1443 - default: - return length - } - - case 1444: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1445 - default: - return length - } - - case 1445: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1446 - default: - return length - } - - case 1446: - switch byteutil.ByteToLower(b) { - case 'w': - length = i + 1 - st = 1447 - default: - return length - } - - case 1448: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1449 - default: - return length - } - - case 1449: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1450 - default: - return length - } - - case 1450: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1451 - default: - return length - } - - case 1451: - switch byteutil.ByteToLower(b) { - case 'y': - st = 1452 - default: - return length - } - - case 1452: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1453 - default: - return length - } - - case 1453: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1454 - default: - return length - } - - case 1454: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1455 - default: - return length - } - - case 1455: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1456 - default: - return length - } - - case 1457: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1458 - default: - return length - } - - case 1458: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1459 - default: - return length - } - - case 1464: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1465 - case 'p': - st = 1466 - default: - return length - } - - case 1466: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 1467 - default: - return length - } - - case 1468: - switch byteutil.ByteToLower(b) { - case 's': - st = 1469 - default: - return length - } - - case 1469: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1470 - default: - return length - } - - case 1470: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1471 - default: - return length - } - - case 1471: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 1472 - default: - return length - } - - case 1478: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1479 - case 'c': - length = i + 1 - st = 1491 - case 'e': - length = i + 1 - st = 1492 - case 'f': - length = i + 1 - st = 1509 - case 'g': - length = i + 1 - st = 1510 - case 'h': - st = 1512 - case 'i': - length = i + 1 - st = 1514 - case 'l': - length = i + 1 - st = 1524 - case 'o': - length = i + 1 - st = 1525 - case 'p': - length = i + 1 - st = 1526 - case 'r': - length = i + 1 - st = 1527 - case 't': - st = 1530 - case 'u': - length = i + 1 - st = 1532 - case 'y': - st = 1533 - case 'z': - length = i + 1 - st = 1535 - default: - return length - } - - case 1479: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1480 - case 'g': - st = 1483 - case 'm': - st = 1487 - case 'v': - st = 1489 - default: - return length - } - - case 1480: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1481 - default: - return length - } - - case 1481: - switch byteutil.ByteToLower(b) { - case 'x': - length = i + 1 - st = 1482 - default: - return length - } - - case 1483: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1484 - default: - return length - } - - case 1484: - switch byteutil.ByteToLower(b) { - case 'y': - st = 1485 - default: - return length - } - - case 1485: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1486 - default: - return length - } - - case 1487: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1488 - default: - return length - } - - case 1489: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1490 - default: - return length - } - - case 1492: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 1493 - case 't': - length = i + 1 - st = 1494 - case 'u': - st = 1499 - case 'w': - length = i + 1 - st = 1504 - case 'x': - st = 1506 - default: - return length - } - - case 1494: - switch byteutil.ByteToLower(b) { - case 'w': - st = 1495 - default: - return length - } - - case 1495: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1496 - default: - return length - } - - case 1496: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1497 - default: - return length - } - - case 1497: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 1498 - default: - return length - } - - case 1499: - switch byteutil.ByteToLower(b) { - case 's': - st = 1500 - default: - return length - } - - case 1500: - switch byteutil.ByteToLower(b) { - case 't': - st = 1501 - default: - return length - } - - case 1501: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1502 - default: - return length - } - - case 1502: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1503 - default: - return length - } - - case 1504: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1505 - default: - return length - } - - case 1506: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1507 - default: - return length - } - - case 1507: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1508 - default: - return length - } - - case 1510: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1511 - default: - return length - } - - case 1512: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 1513 - default: - return length - } - - case 1514: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1515 - case 'n': - st = 1517 - case 's': - st = 1520 - default: - return length - } - - case 1515: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1516 - default: - return length - } - - case 1517: - switch byteutil.ByteToLower(b) { - case 'j': - st = 1518 - default: - return length - } - - case 1518: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1519 - default: - return length - } - - case 1520: - switch byteutil.ByteToLower(b) { - case 's': - st = 1521 - default: - return length - } - - case 1521: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1522 - default: - return length - } - - case 1522: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1523 - default: - return length - } - - case 1527: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1528 - case 'w': - length = i + 1 - st = 1529 - default: - return length - } - - case 1530: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1531 - default: - return length - } - - case 1533: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 1534 - default: - return length - } - - case 1536: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1537 - case 'm': - length = i + 1 - st = 1543 - case 'n': - st = 1544 - case 'o': - st = 1554 - case 'r': - st = 1556 - case 's': - st = 1566 - case 't': - st = 1570 - case 'v': - st = 1575 - default: - return length - } - - case 1537: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1538 - default: - return length - } - - case 1538: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1539 - default: - return length - } - - case 1539: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1540 - default: - return length - } - - case 1540: - switch byteutil.ByteToLower(b) { - case 'w': - st = 1541 - default: - return length - } - - case 1541: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1542 - default: - return length - } - - case 1544: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1545 - case 'g': - length = i + 1 - st = 1546 - case 'i': - st = 1547 - case 'l': - length = i + 1 - st = 1550 - default: - return length - } - - case 1547: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1548 - default: - return length - } - - case 1548: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1549 - default: - return length - } - - case 1550: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1551 - default: - return length - } - - case 1551: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1552 - default: - return length - } - - case 1552: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1553 - default: - return length - } - - case 1554: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1555 - default: - return length - } - - case 1556: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1557 - case 'g': - length = i + 1 - st = 1561 - default: - return length - } - - case 1557: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1558 - default: - return length - } - - case 1558: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1559 - default: - return length - } - - case 1559: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1560 - default: - return length - } - - case 1561: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1562 - default: - return length - } - - case 1562: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1563 - default: - return length - } - - case 1563: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1564 - default: - return length - } - - case 1564: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 1565 - default: - return length - } - - case 1566: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1567 - default: - return length - } - - case 1567: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1568 - default: - return length - } - - case 1568: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1569 - default: - return length - } - - case 1570: - switch byteutil.ByteToLower(b) { - case 's': - st = 1571 - default: - return length - } - - case 1571: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1572 - default: - return length - } - - case 1572: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1573 - default: - return length - } - - case 1573: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1574 - default: - return length - } - - case 1575: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 1576 - default: - return length - } - - case 1577: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1578 - case 'e': - length = i + 1 - st = 1596 - case 'f': - length = i + 1 - st = 1597 - case 'g': - length = i + 1 - st = 1598 - case 'h': - length = i + 1 - st = 1599 - case 'i': - st = 1625 - case 'k': - length = i + 1 - st = 1644 - case 'l': - length = i + 1 - st = 1645 - case 'm': - length = i + 1 - st = 1656 - case 'n': - length = i + 1 - st = 1657 - case 'o': - st = 1658 - case 'r': - length = i + 1 - st = 1668 - case 's': - length = i + 1 - st = 1693 - case 't': - length = i + 1 - st = 1694 - case 'u': - st = 1695 - case 'w': - length = i + 1 - st = 1697 - case 'y': - length = i + 1 - st = 1698 - default: - return length - } - - case 1578: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1579 - case 'n': - st = 1581 - case 'r': - st = 1586 - default: - return length - } - - case 1579: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1580 - default: - return length - } - - case 1581: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1582 - default: - return length - } - - case 1582: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1583 - default: - return length - } - - case 1583: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1584 - default: - return length - } - - case 1584: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1585 - default: - return length - } - - case 1586: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1587 - case 't': - st = 1589 - default: - return length - } - - case 1587: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1588 - default: - return length - } - - case 1589: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1590 - case 's': - length = i + 1 - st = 1594 - case 'y': - length = i + 1 - st = 1595 - default: - return length - } - - case 1590: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1591 - default: - return length - } - - case 1591: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1592 - default: - return length - } - - case 1592: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1593 - default: - return length - } - - case 1599: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1600 - case 'i': - st = 1606 - case 'o': - st = 1611 - case 'y': - st = 1621 - default: - return length - } - - case 1600: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1601 - default: - return length - } - - case 1601: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1602 - default: - return length - } - - case 1602: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1603 - default: - return length - } - - case 1603: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1604 - default: - return length - } - - case 1604: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1605 - default: - return length - } - - case 1606: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1607 - default: - return length - } - - case 1607: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1608 - default: - return length - } - - case 1608: - switch byteutil.ByteToLower(b) { - case 'p': - st = 1609 - default: - return length - } - - case 1609: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1610 - default: - return length - } - - case 1611: - switch byteutil.ByteToLower(b) { - case 't': - st = 1612 - default: - return length - } - - case 1612: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1613 - default: - return length - } - - case 1613: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1614 - case 's': - length = i + 1 - st = 1620 - default: - return length - } - - case 1614: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1615 - default: - return length - } - - case 1615: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1616 - default: - return length - } - - case 1616: - switch byteutil.ByteToLower(b) { - case 'p': - st = 1617 - default: - return length - } - - case 1617: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1618 - default: - return length - } - - case 1618: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1619 - default: - return length - } - - case 1621: - switch byteutil.ByteToLower(b) { - case 's': - st = 1622 - default: - return length - } - - case 1622: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1623 - default: - return length - } - - case 1623: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1624 - default: - return length - } - - case 1625: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1626 - case 'c': - st = 1630 - case 'n': - st = 1639 - case 'z': - st = 1641 - default: - return length - } - - case 1626: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1627 - default: - return length - } - - case 1627: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1628 - default: - return length - } - - case 1628: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1629 - default: - return length - } - - case 1630: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1631 - case 't': - st = 1632 - default: - return length - } - - case 1632: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1633 - case 'u': - st = 1635 - default: - return length - } - - case 1633: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1634 - default: - return length - } - - case 1635: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1636 - default: - return length - } - - case 1636: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1637 - default: - return length - } - - case 1637: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1638 - default: - return length - } - - case 1639: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 1640 - default: - return length - } - - case 1641: - switch byteutil.ByteToLower(b) { - case 'z': - st = 1642 - default: - return length - } - - case 1642: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1643 - default: - return length - } - - case 1645: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1646 - case 'u': - st = 1649 - default: - return length - } - - case 1646: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1647 - default: - return length - } - - case 1647: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1648 - default: - return length - } - - case 1649: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1650 - case 's': - length = i + 1 - st = 1655 - default: - return length - } - - case 1650: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1651 - default: - return length - } - - case 1651: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1652 - default: - return length - } - - case 1652: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1653 - default: - return length - } - - case 1653: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1654 - default: - return length - } - - case 1658: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1659 - case 'k': - st = 1661 - case 'r': - st = 1664 - case 's': - st = 1666 - default: - return length - } - - case 1659: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1660 - default: - return length - } - - case 1661: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1662 - default: - return length - } - - case 1662: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1663 - default: - return length - } - - case 1664: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1665 - default: - return length - } - - case 1666: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1667 - default: - return length - } - - case 1668: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1669 - case 'e': - st = 1672 - case 'o': - length = i + 1 - st = 1675 - default: - return length - } - - case 1669: - switch byteutil.ByteToLower(b) { - case 'x': - st = 1670 - default: - return length - } - - case 1670: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1671 - default: - return length - } - - case 1672: - switch byteutil.ByteToLower(b) { - case 's': - st = 1673 - default: - return length - } - - case 1673: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1674 - default: - return length - } - - case 1675: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 1676 - case 'f': - length = i + 1 - st = 1684 - case 'p': - st = 1685 - default: - return length - } - - case 1676: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1677 - default: - return length - } - - case 1677: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1678 - default: - return length - } - - case 1678: - switch byteutil.ByteToLower(b) { - case 't': - st = 1679 - default: - return length - } - - case 1679: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1680 - default: - return length - } - - case 1680: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1681 - default: - return length - } - - case 1681: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1682 - default: - return length - } - - case 1682: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1683 - default: - return length - } - - case 1685: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1686 - default: - return length - } - - case 1686: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1687 - default: - return length - } - - case 1687: - switch byteutil.ByteToLower(b) { - case 't': - st = 1688 - default: - return length - } - - case 1688: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1689 - case 'y': - length = i + 1 - st = 1692 - default: - return length - } - - case 1689: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1690 - default: - return length - } - - case 1690: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1691 - default: - return length - } - - case 1695: - switch byteutil.ByteToLower(b) { - case 'b': - length = i + 1 - st = 1696 - default: - return length - } - - case 1699: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1700 - case 'p': - st = 1701 - case 'u': - st = 1704 - default: - return length - } - - case 1701: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1702 - default: - return length - } - - case 1702: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1703 - default: - return length - } - - case 1704: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1705 - default: - return length - } - - case 1705: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1706 - default: - return length - } - - case 1706: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1707 - default: - return length - } - - case 1707: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 1708 - default: - return length - } - - case 1709: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1710 - case 'e': - length = i + 1 - st = 1715 - case 'i': - st = 1772 - case 'o': - length = i + 1 - st = 1777 - case 's': - length = i + 1 - st = 1784 - case 'u': - length = i + 1 - st = 1787 - case 'w': - length = i + 1 - st = 1791 - case 'y': - st = 1792 - default: - return length - } - - case 1710: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1711 - default: - return length - } - - case 1711: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1712 - default: - return length - } - - case 1712: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1713 - default: - return length - } - - case 1713: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1714 - default: - return length - } - - case 1715: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1716 - case 'c': - st = 1721 - case 'd': - length = i + 1 - st = 1726 - case 'h': - st = 1732 - case 'i': - st = 1735 - case 'n': - length = i + 1 - st = 1740 - case 'p': - st = 1745 - case 's': - st = 1759 - case 'v': - st = 1767 - default: - return length - } - - case 1716: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1717 - default: - return length - } - - case 1717: - switch byteutil.ByteToLower(b) { - case 't': - st = 1718 - default: - return length - } - - case 1718: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1719 - default: - return length - } - - case 1719: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1720 - default: - return length - } - - case 1721: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1722 - default: - return length - } - - case 1722: - switch byteutil.ByteToLower(b) { - case 'p': - st = 1723 - default: - return length - } - - case 1723: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1724 - default: - return length - } - - case 1724: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1725 - default: - return length - } - - case 1726: - switch byteutil.ByteToLower(b) { - case 's': - st = 1727 - default: - return length - } - - case 1727: - switch byteutil.ByteToLower(b) { - case 't': - st = 1728 - default: - return length - } - - case 1728: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1729 - default: - return length - } - - case 1729: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1730 - default: - return length - } - - case 1730: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1731 - default: - return length - } - - case 1732: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1733 - default: - return length - } - - case 1733: - switch byteutil.ByteToLower(b) { - case 'b': - length = i + 1 - st = 1734 - default: - return length - } - - case 1735: - switch byteutil.ByteToLower(b) { - case 's': - st = 1736 - case 't': - length = i + 1 - st = 1739 - default: - return length - } - - case 1736: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1737 - default: - return length - } - - case 1737: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1738 - default: - return length - } - - case 1740: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1741 - default: - return length - } - - case 1741: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1742 - default: - return length - } - - case 1742: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1743 - default: - return length - } - - case 1743: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1744 - default: - return length - } - - case 1745: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1746 - case 'o': - st = 1749 - case 'u': - st = 1752 - default: - return length - } - - case 1746: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1747 - default: - return length - } - - case 1747: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1748 - default: - return length - } - - case 1749: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1750 - default: - return length - } - - case 1750: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1751 - default: - return length - } - - case 1752: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1753 - default: - return length - } - - case 1753: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1754 - default: - return length - } - - case 1754: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1755 - default: - return length - } - - case 1755: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1756 - default: - return length - } - - case 1756: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1757 - default: - return length - } - - case 1757: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 1758 - default: - return length - } - - case 1759: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1760 - default: - return length - } - - case 1760: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1761 - default: - return length - } - - case 1761: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1762 - default: - return length - } - - case 1762: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1763 - default: - return length - } - - case 1763: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1764 - default: - return length - } - - case 1764: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1765 - default: - return length - } - - case 1765: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1766 - default: - return length - } - - case 1767: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1768 - default: - return length - } - - case 1768: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1769 - default: - return length - } - - case 1769: - switch byteutil.ByteToLower(b) { - case 'w': - length = i + 1 - st = 1770 - default: - return length - } - - case 1770: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1771 - default: - return length - } - - case 1772: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1773 - case 'o': - length = i + 1 - st = 1775 - case 'p': - length = i + 1 - st = 1776 - default: - return length - } - - case 1773: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 1774 - default: - return length - } - - case 1777: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1778 - case 'd': - st = 1781 - default: - return length - } - - case 1778: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1779 - default: - return length - } - - case 1779: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1780 - default: - return length - } - - case 1781: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1782 - default: - return length - } - - case 1782: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1783 - default: - return length - } - - case 1784: - switch byteutil.ByteToLower(b) { - case 'v': - st = 1785 - default: - return length - } - - case 1785: - switch byteutil.ByteToLower(b) { - case 'p': - length = i + 1 - st = 1786 - default: - return length - } - - case 1787: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1788 - case 'n': - length = i + 1 - st = 1790 - default: - return length - } - - case 1788: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1789 - default: - return length - } - - case 1792: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1793 - default: - return length - } - - case 1793: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1794 - default: - return length - } - - case 1794: - switch byteutil.ByteToLower(b) { - case 'y': - st = 1795 - default: - return length - } - - case 1795: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 1796 - default: - return length - } - - case 1797: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1798 - case 'b': - length = i + 1 - st = 1830 - case 'c': - length = i + 1 - st = 1831 - case 'd': - length = i + 1 - st = 1864 - case 'e': - length = i + 1 - st = 1865 - case 'g': - length = i + 1 - st = 1880 - case 'h': - length = i + 1 - st = 1881 - case 'i': - length = i + 1 - st = 1896 - case 'j': - length = i + 1 - st = 1904 - case 'k': - length = i + 1 - st = 1905 - case 'l': - length = i + 1 - st = 1908 - case 'm': - length = i + 1 - st = 1909 - case 'n': - length = i + 1 - st = 1910 - case 'o': - length = i + 1 - st = 1913 - case 'p': - st = 1941 - case 'r': - length = i + 1 - st = 1961 - case 't': - length = i + 1 - st = 1962 - case 'u': - length = i + 1 - st = 1969 - case 'v': - length = i + 1 - st = 1993 - case 'w': - st = 1994 - case 'x': - length = i + 1 - st = 1998 - case 'y': - length = i + 1 - st = 1999 - case 'z': - length = i + 1 - st = 2009 - default: - return length - } - - case 1798: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1799 - case 'l': - st = 1805 - case 'm': - st = 1807 - case 'n': - st = 1812 - case 'p': - length = i + 1 - st = 1825 - case 'r': - st = 1826 - case 'x': - st = 1828 - default: - return length - } - - case 1799: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1800 - default: - return length - } - - case 1800: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1801 - default: - return length - } - - case 1801: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1802 - default: - return length - } - - case 1802: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1803 - default: - return length - } - - case 1803: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 1804 - default: - return length - } - - case 1805: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1806 - default: - return length - } - - case 1807: - switch byteutil.ByteToLower(b) { - case 's': - st = 1808 - default: - return length - } - - case 1808: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1809 - default: - return length - } - - case 1809: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1810 - default: - return length - } - - case 1810: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1811 - default: - return length - } - - case 1812: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1813 - default: - return length - } - - case 1813: - switch byteutil.ByteToLower(b) { - case 'v': - st = 1814 - default: - return length - } - - case 1814: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1815 - default: - return length - } - - case 1815: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 1816 - default: - return length - } - - case 1816: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1817 - default: - return length - } - - case 1817: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1818 - default: - return length - } - - case 1818: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1819 - default: - return length - } - - case 1819: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1820 - default: - return length - } - - case 1820: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1821 - default: - return length - } - - case 1821: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1822 - default: - return length - } - - case 1822: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1823 - default: - return length - } - - case 1823: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1824 - default: - return length - } - - case 1826: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1827 - default: - return length - } - - case 1828: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 1829 - default: - return length - } - - case 1831: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1832 - case 'b': - length = i + 1 - st = 1833 - case 'h': - st = 1834 - case 'i': - st = 1857 - case 'o': - st = 1862 - default: - return length - } - - case 1834: - switch byteutil.ByteToLower(b) { - case 'm': - st = 1835 - case 'o': - st = 1839 - case 'u': - st = 1850 - case 'w': - st = 1853 - default: - return length - } - - case 1835: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1836 - default: - return length - } - - case 1836: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1837 - default: - return length - } - - case 1837: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1838 - default: - return length - } - - case 1839: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1840 - case 'o': - st = 1848 - default: - return length - } - - case 1840: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1841 - default: - return length - } - - case 1841: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1842 - default: - return length - } - - case 1842: - switch byteutil.ByteToLower(b) { - case 's': - st = 1843 - default: - return length - } - - case 1843: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1844 - default: - return length - } - - case 1844: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1845 - default: - return length - } - - case 1845: - switch byteutil.ByteToLower(b) { - case 'p': - st = 1846 - default: - return length - } - - case 1846: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1847 - default: - return length - } - - case 1848: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1849 - default: - return length - } - - case 1850: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1851 - default: - return length - } - - case 1851: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1852 - default: - return length - } - - case 1853: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1854 - default: - return length - } - - case 1854: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1855 - default: - return length - } - - case 1855: - switch byteutil.ByteToLower(b) { - case 'z': - length = i + 1 - st = 1856 - default: - return length - } - - case 1857: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1858 - default: - return length - } - - case 1858: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1859 - default: - return length - } - - case 1859: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1860 - default: - return length - } - - case 1860: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1861 - default: - return length - } - - case 1862: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1863 - default: - return length - } - - case 1865: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1866 - case 'n': - st = 1868 - case 'r': - st = 1871 - case 'w': - length = i + 1 - st = 1877 - case 'x': - length = i + 1 - st = 1878 - default: - return length - } - - case 1866: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1867 - default: - return length - } - - case 1868: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1869 - default: - return length - } - - case 1869: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1870 - default: - return length - } - - case 1871: - switch byteutil.ByteToLower(b) { - case 'v': - st = 1872 - default: - return length - } - - case 1872: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1873 - default: - return length - } - - case 1873: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1874 - default: - return length - } - - case 1874: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1875 - default: - return length - } - - case 1875: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1876 - default: - return length - } - - case 1878: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1879 - default: - return length - } - - case 1881: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1882 - case 'o': - st = 1887 - case 'r': - st = 1891 - default: - return length - } - - case 1882: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1883 - default: - return length - } - - case 1883: - switch byteutil.ByteToLower(b) { - case 's': - st = 1884 - default: - return length - } - - case 1884: - switch byteutil.ByteToLower(b) { - case 'h': - st = 1885 - default: - return length - } - - case 1885: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 1886 - default: - return length - } - - case 1887: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1888 - case 'w': - length = i + 1 - st = 1890 - default: - return length - } - - case 1888: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1889 - default: - return length - } - - case 1891: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1892 - default: - return length - } - - case 1892: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1893 - default: - return length - } - - case 1893: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1894 - default: - return length - } - - case 1894: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 1895 - default: - return length - } - - case 1896: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1897 - case 't': - st = 1902 - default: - return length - } - - case 1897: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1898 - default: - return length - } - - case 1898: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1899 - default: - return length - } - - case 1899: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1900 - default: - return length - } - - case 1900: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1901 - default: - return length - } - - case 1902: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1903 - default: - return length - } - - case 1905: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1906 - case 'y': - length = i + 1 - st = 1907 - default: - return length - } - - case 1910: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1911 - default: - return length - } - - case 1911: - switch byteutil.ByteToLower(b) { - case 'f': - length = i + 1 - st = 1912 - default: - return length - } - - case 1913: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1914 - case 'f': - st = 1921 - case 'h': - st = 1927 - case 'l': - st = 1929 - case 'n': - st = 1938 - case 'y': - length = i + 1 - st = 1940 - default: - return length - } - - case 1914: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1915 - case 'i': - st = 1918 - default: - return length - } - - case 1915: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1916 - default: - return length - } - - case 1916: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1917 - default: - return length - } - - case 1918: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1919 - default: - return length - } - - case 1919: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1920 - default: - return length - } - - case 1921: - switch byteutil.ByteToLower(b) { - case 't': - st = 1922 - default: - return length - } - - case 1922: - switch byteutil.ByteToLower(b) { - case 'w': - st = 1923 - default: - return length - } - - case 1923: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1924 - default: - return length - } - - case 1924: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1925 - default: - return length - } - - case 1925: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1926 - default: - return length - } - - case 1927: - switch byteutil.ByteToLower(b) { - case 'u': - length = i + 1 - st = 1928 - default: - return length - } - - case 1929: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1930 - case 'u': - st = 1932 - default: - return length - } - - case 1930: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 1931 - default: - return length - } - - case 1932: - switch byteutil.ByteToLower(b) { - case 't': - st = 1933 - default: - return length - } - - case 1933: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1934 - default: - return length - } - - case 1934: - switch byteutil.ByteToLower(b) { - case 'o': - st = 1935 - default: - return length - } - - case 1935: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1936 - default: - return length - } - - case 1936: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1937 - default: - return length - } - - case 1938: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1939 - default: - return length - } - - case 1941: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1942 - case 'i': - st = 1945 - case 'r': - st = 1950 - default: - return length - } - - case 1942: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1943 - default: - return length - } - - case 1943: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1944 - default: - return length - } - - case 1945: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1946 - default: - return length - } - - case 1946: - switch byteutil.ByteToLower(b) { - case 'g': - st = 1947 - default: - return length - } - - case 1947: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1948 - default: - return length - } - - case 1948: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 1949 - default: - return length - } - - case 1950: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1951 - default: - return length - } - - case 1951: - switch byteutil.ByteToLower(b) { - case 'a': - st = 1952 - default: - return length - } - - case 1952: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1953 - default: - return length - } - - case 1953: - switch byteutil.ByteToLower(b) { - case 'b': - st = 1954 - default: - return length - } - - case 1954: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1955 - default: - return length - } - - case 1955: - switch byteutil.ByteToLower(b) { - case 't': - st = 1956 - default: - return length - } - - case 1956: - switch byteutil.ByteToLower(b) { - case 't': - st = 1957 - default: - return length - } - - case 1957: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1958 - default: - return length - } - - case 1958: - switch byteutil.ByteToLower(b) { - case 'n': - st = 1959 - default: - return length - } - - case 1959: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 1960 - default: - return length - } - - case 1962: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1963 - case 'y': - st = 1966 - default: - return length - } - - case 1963: - switch byteutil.ByteToLower(b) { - case 'd': - st = 1964 - default: - return length - } - - case 1964: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1965 - default: - return length - } - - case 1966: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1967 - default: - return length - } - - case 1967: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 1968 - default: - return length - } - - case 1969: - switch byteutil.ByteToLower(b) { - case 'c': - st = 1970 - case 'p': - st = 1973 - case 'r': - st = 1983 - case 'z': - st = 1989 - default: - return length - } - - case 1970: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1971 - default: - return length - } - - case 1971: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1972 - default: - return length - } - - case 1973: - switch byteutil.ByteToLower(b) { - case 'p': - st = 1974 - default: - return length - } - - case 1974: - switch byteutil.ByteToLower(b) { - case 'l': - st = 1975 - case 'o': - st = 1980 - default: - return length - } - - case 1975: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1976 - case 'y': - length = i + 1 - st = 1979 - default: - return length - } - - case 1976: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1977 - default: - return length - } - - case 1977: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1978 - default: - return length - } - - case 1980: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1981 - default: - return length - } - - case 1981: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 1982 - default: - return length - } - - case 1983: - switch byteutil.ByteToLower(b) { - case 'f': - length = i + 1 - st = 1984 - case 'g': - st = 1985 - default: - return length - } - - case 1985: - switch byteutil.ByteToLower(b) { - case 'e': - st = 1986 - default: - return length - } - - case 1986: - switch byteutil.ByteToLower(b) { - case 'r': - st = 1987 - default: - return length - } - - case 1987: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 1988 - default: - return length - } - - case 1989: - switch byteutil.ByteToLower(b) { - case 'u': - st = 1990 - default: - return length - } - - case 1990: - switch byteutil.ByteToLower(b) { - case 'k': - st = 1991 - default: - return length - } - - case 1991: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 1992 - default: - return length - } - - case 1994: - switch byteutil.ByteToLower(b) { - case 'i': - st = 1995 - default: - return length - } - - case 1995: - switch byteutil.ByteToLower(b) { - case 's': - st = 1996 - default: - return length - } - - case 1996: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 1997 - default: - return length - } - - case 1999: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2000 - case 's': - st = 2004 - default: - return length - } - - case 2000: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2001 - default: - return length - } - - case 2001: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2002 - default: - return length - } - - case 2002: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 2003 - default: - return length - } - - case 2004: - switch byteutil.ByteToLower(b) { - case 't': - st = 2005 - default: - return length - } - - case 2005: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2006 - default: - return length - } - - case 2006: - switch byteutil.ByteToLower(b) { - case 'm': - st = 2007 - default: - return length - } - - case 2007: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2008 - default: - return length - } - - case 2010: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2011 - case 'c': - length = i + 1 - st = 2024 - case 'd': - length = i + 1 - st = 2025 - case 'e': - st = 2026 - case 'f': - length = i + 1 - st = 2047 - case 'g': - length = i + 1 - st = 2048 - case 'h': - length = i + 1 - st = 2049 - case 'i': - st = 2056 - case 'j': - length = i + 1 - st = 2073 - case 'k': - length = i + 1 - st = 2074 - case 'l': - length = i + 1 - st = 2075 - case 'm': - length = i + 1 - st = 2076 - case 'n': - length = i + 1 - st = 2077 - case 'o': - length = i + 1 - st = 2078 - case 'r': - length = i + 1 - st = 2104 - case 't': - length = i + 1 - st = 2122 - case 'u': - st = 2123 - case 'v': - length = i + 1 - st = 2125 - case 'w': - length = i + 1 - st = 2126 - case 'z': - length = i + 1 - st = 2127 - default: - return length - } - - case 2011: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2012 - case 't': - st = 2016 - case 'x': - length = i + 1 - st = 2022 - default: - return length - } - - case 2012: - switch byteutil.ByteToLower(b) { - case 'p': - st = 2013 - default: - return length - } - - case 2013: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2014 - default: - return length - } - - case 2014: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 2015 - default: - return length - } - - case 2016: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2017 - case 't': - st = 2019 - default: - return length - } - - case 2017: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 2018 - default: - return length - } - - case 2019: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2020 - default: - return length - } - - case 2020: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 2021 - default: - return length - } - - case 2022: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 2023 - default: - return length - } - - case 2026: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2027 - case 'c': - st = 2029 - case 'l': - length = i + 1 - st = 2037 - case 'm': - st = 2038 - case 'n': - st = 2043 - default: - return length - } - - case 2027: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 2028 - default: - return length - } - - case 2029: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 2030 - default: - return length - } - - case 2030: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2031 - default: - return length - } - - case 2031: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2032 - default: - return length - } - - case 2032: - switch byteutil.ByteToLower(b) { - case 'l': - st = 2033 - default: - return length - } - - case 2033: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2034 - default: - return length - } - - case 2034: - switch byteutil.ByteToLower(b) { - case 'g': - st = 2035 - default: - return length - } - - case 2035: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 2036 - default: - return length - } - - case 2038: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2039 - default: - return length - } - - case 2039: - switch byteutil.ByteToLower(b) { - case 's': - st = 2040 - default: - return length - } - - case 2040: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2041 - default: - return length - } - - case 2041: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 2042 - default: - return length - } - - case 2043: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2044 - default: - return length - } - - case 2044: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2045 - default: - return length - } - - case 2045: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2046 - default: - return length - } - - case 2049: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 2050 - case 'e': - st = 2051 - default: - return length - } - - case 2051: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2052 - default: - return length - } - - case 2052: - switch byteutil.ByteToLower(b) { - case 't': - st = 2053 - default: - return length - } - - case 2053: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2054 - default: - return length - } - - case 2054: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 2055 - default: - return length - } - - case 2056: - switch byteutil.ByteToLower(b) { - case 'c': - st = 2057 - case 'e': - st = 2062 - case 'p': - st = 2066 - case 'r': - st = 2068 - default: - return length - } - - case 2057: - switch byteutil.ByteToLower(b) { - case 'k': - st = 2058 - default: - return length - } - - case 2058: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2059 - default: - return length - } - - case 2059: - switch byteutil.ByteToLower(b) { - case 't': - st = 2060 - default: - return length - } - - case 2060: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2061 - default: - return length - } - - case 2062: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2063 - default: - return length - } - - case 2063: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2064 - default: - return length - } - - case 2064: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 2065 - default: - return length - } - - case 2066: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2067 - default: - return length - } - - case 2068: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2069 - case 'o': - st = 2071 - default: - return length - } - - case 2069: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2070 - default: - return length - } - - case 2071: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 2072 - default: - return length - } - - case 2078: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2079 - case 'k': - st = 2082 - case 'o': - st = 2085 - case 'p': - length = i + 1 - st = 2088 - case 'r': - st = 2089 - case 's': - st = 2092 - case 'u': - st = 2097 - case 'w': - st = 2100 - case 'y': - st = 2102 - default: - return length - } - - case 2079: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2080 - default: - return length - } - - case 2080: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 2081 - default: - return length - } - - case 2082: - switch byteutil.ByteToLower(b) { - case 'y': - st = 2083 - default: - return length - } - - case 2083: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 2084 - default: - return length - } - - case 2085: - switch byteutil.ByteToLower(b) { - case 'l': - st = 2086 - default: - return length - } - - case 2086: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2087 - default: - return length - } - - case 2089: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2090 - default: - return length - } - - case 2090: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 2091 - default: - return length - } - - case 2092: - switch byteutil.ByteToLower(b) { - case 'h': - st = 2093 - default: - return length - } - - case 2093: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2094 - default: - return length - } - - case 2094: - switch byteutil.ByteToLower(b) { - case 'b': - st = 2095 - default: - return length - } - - case 2095: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 2096 - default: - return length - } - - case 2097: - switch byteutil.ByteToLower(b) { - case 'r': - st = 2098 - default: - return length - } - - case 2098: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2099 - default: - return length - } - - case 2100: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 2101 - default: - return length - } - - case 2102: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2103 - default: - return length - } - - case 2104: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2105 - case 'u': - st = 2119 - default: - return length - } - - case 2105: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2106 - case 'i': - st = 2111 - case 'v': - st = 2116 - default: - return length - } - - case 2106: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 2107 - case 'i': - st = 2108 - default: - return length - } - - case 2108: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2109 - default: - return length - } - - case 2109: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 2110 - default: - return length - } - - case 2111: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2112 - default: - return length - } - - case 2112: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2113 - default: - return length - } - - case 2113: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2114 - default: - return length - } - - case 2114: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 2115 - default: - return length - } - - case 2116: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2117 - default: - return length - } - - case 2117: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 2118 - default: - return length - } - - case 2119: - switch byteutil.ByteToLower(b) { - case 's': - st = 2120 - default: - return length - } - - case 2120: - switch byteutil.ByteToLower(b) { - case 't': - length = i + 1 - st = 2121 - default: - return length - } - - case 2123: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 2124 - default: - return length - } - - case 2128: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 2129 - case 'g': - length = i + 1 - st = 2130 - case 'k': - length = i + 1 - st = 2131 - case 'n': - st = 2132 - case 'o': - st = 2142 - case 's': - length = i + 1 - st = 2144 - case 'y': - length = i + 1 - st = 2145 - case 'z': - length = i + 1 - st = 2146 - default: - return length - } - - case 2132: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2133 - case 'o': - length = i + 1 - st = 2141 - default: - return length - } - - case 2133: - switch byteutil.ByteToLower(b) { - case 'v': - st = 2134 - default: - return length - } - - case 2134: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2135 - default: - return length - } - - case 2135: - switch byteutil.ByteToLower(b) { - case 'r': - st = 2136 - default: - return length - } - - case 2136: - switch byteutil.ByteToLower(b) { - case 's': - st = 2137 - default: - return length - } - - case 2137: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2138 - default: - return length - } - - case 2138: - switch byteutil.ByteToLower(b) { - case 't': - st = 2139 - default: - return length - } - - case 2139: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 2140 - default: - return length - } - - case 2142: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 2143 - default: - return length - } - - case 2147: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 2148 - case 'c': - length = i + 1 - st = 2156 - case 'e': - length = i + 1 - st = 2157 - case 'g': - length = i + 1 - st = 2178 - case 'i': - length = i + 1 - st = 2179 - case 'l': - st = 2195 - case 'n': - length = i + 1 - st = 2204 - case 'o': - st = 2205 - case 'u': - length = i + 1 - st = 2219 - default: - return length - } - - case 2148: - switch byteutil.ByteToLower(b) { - case 'c': - st = 2149 - default: - return length - } - - case 2149: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2150 - default: - return length - } - - case 2150: - switch byteutil.ByteToLower(b) { - case 't': - st = 2151 - default: - return length - } - - case 2151: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2152 - default: - return length - } - - case 2152: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2153 - default: - return length - } - - case 2153: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2154 - default: - return length - } - - case 2154: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2155 - default: - return length - } - - case 2157: - switch byteutil.ByteToLower(b) { - case 'g': - st = 2158 - case 'n': - st = 2161 - case 'r': - st = 2167 - case 't': - length = i + 1 - st = 2177 - default: - return length - } - - case 2158: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2159 - default: - return length - } - - case 2159: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2160 - default: - return length - } - - case 2161: - switch byteutil.ByteToLower(b) { - case 't': - st = 2162 - default: - return length - } - - case 2162: - switch byteutil.ByteToLower(b) { - case 'u': - st = 2163 - default: - return length - } - - case 2163: - switch byteutil.ByteToLower(b) { - case 'r': - st = 2164 - default: - return length - } - - case 2164: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2165 - default: - return length - } - - case 2165: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2166 - default: - return length - } - - case 2167: - switch byteutil.ByteToLower(b) { - case 's': - st = 2168 - default: - return length - } - - case 2168: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2169 - default: - return length - } - - case 2169: - switch byteutil.ByteToLower(b) { - case 'c': - st = 2170 - default: - return length - } - - case 2170: - switch byteutil.ByteToLower(b) { - case 'h': - st = 2171 - default: - return length - } - - case 2171: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2172 - default: - return length - } - - case 2172: - switch byteutil.ByteToLower(b) { - case 'r': - st = 2173 - default: - return length - } - - case 2173: - switch byteutil.ByteToLower(b) { - case 'u': - st = 2174 - default: - return length - } - - case 2174: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2175 - default: - return length - } - - case 2175: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 2176 - default: - return length - } - - case 2179: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2180 - case 'd': - st = 2184 - case 'l': - st = 2187 - case 's': - st = 2191 - default: - return length - } - - case 2180: - switch byteutil.ByteToLower(b) { - case 'j': - st = 2181 - default: - return length - } - - case 2181: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2182 - default: - return length - } - - case 2182: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2183 - default: - return length - } - - case 2184: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2185 - default: - return length - } - - case 2185: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 2186 - default: - return length - } - - case 2187: - switch byteutil.ByteToLower(b) { - case 'l': - st = 2188 - default: - return length - } - - case 2188: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2189 - default: - return length - } - - case 2189: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2190 - default: - return length - } - - case 2191: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2192 - default: - return length - } - - case 2192: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2193 - default: - return length - } - - case 2193: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 2194 - default: - return length - } - - case 2195: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2196 - default: - return length - } - - case 2196: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2197 - default: - return length - } - - case 2197: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2198 - default: - return length - } - - case 2198: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2199 - default: - return length - } - - case 2199: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2200 - default: - return length - } - - case 2200: - switch byteutil.ByteToLower(b) { - case 'r': - st = 2201 - default: - return length - } - - case 2201: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2202 - default: - return length - } - - case 2202: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 2203 - default: - return length - } - - case 2205: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2206 - case 't': - st = 2209 - case 'y': - st = 2215 - default: - return length - } - - case 2206: - switch byteutil.ByteToLower(b) { - case 'k': - st = 2207 - default: - return length - } - - case 2207: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 2208 - default: - return length - } - - case 2209: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 2210 - case 'i': - st = 2211 - case 'o': - length = i + 1 - st = 2214 - default: - return length - } - - case 2211: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2212 - default: - return length - } - - case 2212: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 2213 - default: - return length - } - - case 2215: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2216 - default: - return length - } - - case 2216: - switch byteutil.ByteToLower(b) { - case 'g': - st = 2217 - default: - return length - } - - case 2217: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 2218 - default: - return length - } - - case 2220: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2221 - case 'e': - st = 2233 - case 'f': - length = i + 1 - st = 2249 - case 'h': - st = 2250 - case 'i': - st = 2256 - case 'm': - st = 2271 - case 'o': - st = 2273 - case 's': - length = i + 1 - st = 2279 - case 't': - st = 2280 - default: - return length - } - - case 2221: - switch byteutil.ByteToLower(b) { - case 'l': - st = 2222 - case 'n': - st = 2228 - case 't': - st = 2230 - default: - return length - } - - case 2222: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2223 - case 't': - st = 2225 - default: - return length - } - - case 2223: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2224 - default: - return length - } - - case 2225: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2226 - default: - return length - } - - case 2226: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 2227 - default: - return length - } - - case 2228: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 2229 - default: - return length - } - - case 2230: - switch byteutil.ByteToLower(b) { - case 'c': - st = 2231 - default: - return length - } - - case 2231: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 2232 - default: - return length - } - - case 2233: - switch byteutil.ByteToLower(b) { - case 'b': - st = 2234 - case 'd': - length = i + 1 - st = 2242 - case 'i': - st = 2247 - default: - return length - } - - case 2234: - switch byteutil.ByteToLower(b) { - case 'c': - st = 2235 - case 's': - st = 2238 - default: - return length - } - - case 2235: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2236 - default: - return length - } - - case 2236: - switch byteutil.ByteToLower(b) { - case 'm': - length = i + 1 - st = 2237 - default: - return length - } - - case 2238: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2239 - default: - return length - } - - case 2239: - switch byteutil.ByteToLower(b) { - case 't': - st = 2240 - default: - return length - } - - case 2240: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 2241 - default: - return length - } - - case 2242: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2243 - default: - return length - } - - case 2243: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2244 - default: - return length - } - - case 2244: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2245 - default: - return length - } - - case 2245: - switch byteutil.ByteToLower(b) { - case 'g': - length = i + 1 - st = 2246 - default: - return length - } - - case 2247: - switch byteutil.ByteToLower(b) { - case 'r': - length = i + 1 - st = 2248 - default: - return length - } - - case 2250: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2251 - default: - return length - } - - case 2251: - switch byteutil.ByteToLower(b) { - case 's': - st = 2252 - default: - return length - } - - case 2252: - switch byteutil.ByteToLower(b) { - case 'w': - st = 2253 - default: - return length - } - - case 2253: - switch byteutil.ByteToLower(b) { - case 'h': - st = 2254 - default: - return length - } - - case 2254: - switch byteutil.ByteToLower(b) { - case 'o': - length = i + 1 - st = 2255 - default: - return length - } - - case 2256: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2257 - case 'k': - st = 2259 - case 'l': - st = 2261 - case 'n': - length = i + 1 - st = 2270 - default: - return length - } - - case 2257: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 2258 - default: - return length - } - - case 2259: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 2260 - default: - return length - } - - case 2261: - switch byteutil.ByteToLower(b) { - case 'l': - st = 2262 - default: - return length - } - - case 2262: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2263 - default: - return length - } - - case 2263: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2264 - default: - return length - } - - case 2264: - switch byteutil.ByteToLower(b) { - case 'm': - st = 2265 - default: - return length - } - - case 2265: - switch byteutil.ByteToLower(b) { - case 'h': - st = 2266 - default: - return length - } - - case 2266: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2267 - default: - return length - } - - case 2267: - switch byteutil.ByteToLower(b) { - case 'l': - st = 2268 - default: - return length - } - - case 2268: - switch byteutil.ByteToLower(b) { - case 'l': - length = i + 1 - st = 2269 - default: - return length - } - - case 2271: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 2272 - default: - return length - } - - case 2273: - switch byteutil.ByteToLower(b) { - case 'r': - st = 2274 - default: - return length - } - - case 2274: - switch byteutil.ByteToLower(b) { - case 'k': - length = i + 1 - st = 2275 - case 'l': - st = 2277 - default: - return length - } - - case 2275: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2276 - default: - return length - } - - case 2277: - switch byteutil.ByteToLower(b) { - case 'd': - length = i + 1 - st = 2278 - default: - return length - } - - case 2280: - switch byteutil.ByteToLower(b) { - case 'c': - length = i + 1 - st = 2281 - case 'f': - length = i + 1 - st = 2282 - default: - return length - } - - case 2283: - switch byteutil.ByteToLower(b) { - case 'b': - st = 2284 - case 'e': - st = 2287 - case 'i': - st = 2291 - case 'x': - st = 2293 - case 'y': - st = 2295 - case 'n': - st = 2860 - default: - return length - } - - case 2284: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2285 - default: - return length - } - - case 2285: - switch byteutil.ByteToLower(b) { - case 'x': - length = i + 1 - st = 2286 - default: - return length - } - - case 2287: - switch byteutil.ByteToLower(b) { - case 'r': - st = 2288 - default: - return length - } - - case 2288: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2289 - default: - return length - } - - case 2289: - switch byteutil.ByteToLower(b) { - case 'x': - length = i + 1 - st = 2290 - default: - return length - } - - case 2291: - switch byteutil.ByteToLower(b) { - case 'n': - length = i + 1 - st = 2292 - default: - return length - } - - case 2293: - switch byteutil.ByteToLower(b) { - case 'x': - length = i + 1 - st = 2294 - default: - return length - } - - case 2295: - switch byteutil.ByteToLower(b) { - case 'z': - length = i + 1 - st = 2296 - default: - return length - } - - case 2297: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2298 - case 'e': - length = i + 1 - st = 2307 - case 'o': - st = 2308 - case 't': - length = i + 1 - st = 2329 - default: - return length - } - - case 2298: - switch byteutil.ByteToLower(b) { - case 'c': - st = 2299 - case 'n': - st = 2303 - default: - return length - } - - case 2299: - switch byteutil.ByteToLower(b) { - case 'h': - st = 2300 - default: - return length - } - - case 2300: - switch byteutil.ByteToLower(b) { - case 't': - st = 2301 - default: - return length - } - - case 2301: - switch byteutil.ByteToLower(b) { - case 's': - length = i + 1 - st = 2302 - default: - return length - } - - case 2303: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2304 - default: - return length - } - - case 2304: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2305 - default: - return length - } - - case 2305: - switch byteutil.ByteToLower(b) { - case 'x': - length = i + 1 - st = 2306 - default: - return length - } - - case 2308: - switch byteutil.ByteToLower(b) { - case 'd': - st = 2309 - case 'g': - st = 2316 - case 'k': - st = 2318 - case 'u': - st = 2324 - default: - return length - } - - case 2309: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2310 - default: - return length - } - - case 2310: - switch byteutil.ByteToLower(b) { - case 'b': - st = 2311 - default: - return length - } - - case 2311: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2312 - default: - return length - } - - case 2312: - switch byteutil.ByteToLower(b) { - case 's': - st = 2313 - default: - return length - } - - case 2313: - switch byteutil.ByteToLower(b) { - case 'h': - st = 2314 - default: - return length - } - - case 2314: - switch byteutil.ByteToLower(b) { - case 'i': - length = i + 1 - st = 2315 - default: - return length - } - - case 2316: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 2317 - default: - return length - } - - case 2318: - switch byteutil.ByteToLower(b) { - case 'o': - st = 2319 - default: - return length - } - - case 2319: - switch byteutil.ByteToLower(b) { - case 'h': - st = 2320 - default: - return length - } - - case 2320: - switch byteutil.ByteToLower(b) { - case 'a': - st = 2321 - default: - return length - } - - case 2321: - switch byteutil.ByteToLower(b) { - case 'm': - st = 2322 - default: - return length - } - - case 2322: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 2323 - default: - return length - } - - case 2324: - switch byteutil.ByteToLower(b) { - case 't': - st = 2325 - default: - return length - } - - case 2325: - switch byteutil.ByteToLower(b) { - case 'u': - st = 2326 - default: - return length - } - - case 2326: - switch byteutil.ByteToLower(b) { - case 'b': - st = 2327 - default: - return length - } - - case 2327: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 2328 - default: - return length - } - - case 2330: - switch byteutil.ByteToLower(b) { - case 'a': - length = i + 1 - st = 2331 - case 'i': - st = 2332 - case 'k': - st = 2334 - case 'm': - length = i + 1 - st = 2337 - case 'o': - st = 2338 - case 'u': - st = 2341 - case 'w': - length = i + 1 - st = 2347 - default: - return length - } - - case 2332: - switch byteutil.ByteToLower(b) { - case 'p': - length = i + 1 - st = 2333 - default: - return length - } - - case 2334: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2335 - default: - return length - } - - case 2335: - switch byteutil.ByteToLower(b) { - case 'y': - length = i + 1 - st = 2336 - default: - return length - } - - case 2338: - switch byteutil.ByteToLower(b) { - case 'n': - st = 2339 - default: - return length - } - - case 2339: - switch byteutil.ByteToLower(b) { - case 'e': - length = i + 1 - st = 2340 - default: - return length - } - - case 2341: - switch byteutil.ByteToLower(b) { - case 'e': - st = 2342 - default: - return length - } - - case 2342: - switch byteutil.ByteToLower(b) { - case 'r': - st = 2343 - default: - return length - } - - case 2343: - switch byteutil.ByteToLower(b) { - case 'i': - st = 2344 - default: - return length - } - - case 2344: - switch byteutil.ByteToLower(b) { - case 'c': - st = 2345 - default: - return length - } - - case 2345: - switch byteutil.ByteToLower(b) { - case 'h': - length = i + 1 - st = 2346 - default: - return length - } - - case 2348: - switch b { - case '.': - length = i + 1 - st = 2349 - default: - return length - } - - case 2350: - switch b { - case '.': - length = i + 1 - st = 2351 - case '0': - st = 2368 - case '1': - st = 2370 - case '2': - st = 2372 - case '3': - st = 2374 - case '4': - st = 2376 - case '5': - st = 2378 - case '6': - st = 2380 - case '7': - st = 2382 - case '8': - st = 2384 - case '9': - st = 2386 - default: - return length - } - - case 2352: - switch b { - case '.': - length = i + 1 - st = 2353 - case '0': - st = 2388 - case '1': - st = 2390 - case '2': - st = 2392 - case '3': - st = 2394 - case '4': - st = 2396 - case '5': - st = 2398 - case '6': - st = 2400 - case '7': - st = 2402 - case '8': - st = 2404 - case '9': - st = 2406 - default: - return length - } - - case 2354: - switch b { - case '.': - length = i + 1 - st = 2355 - case '0': - st = 2408 - case '1': - st = 2410 - case '2': - st = 2412 - case '3': - st = 2414 - case '4': - st = 2416 - case '5': - st = 2418 - case '6': - st = 2420 - case '7': - st = 2422 - case '8': - st = 2424 - case '9': - st = 2426 - default: - return length - } - - case 2356: - switch b { - case '.': - length = i + 1 - st = 2357 - case '0': - st = 2428 - case '1': - st = 2430 - case '2': - st = 2432 - case '3': - st = 2434 - case '4': - st = 2436 - case '5': - st = 2438 - case '6': - st = 2440 - case '7': - st = 2442 - case '8': - st = 2444 - case '9': - st = 2446 - default: - return length - } - - case 2358: - switch b { - case '.': - length = i + 1 - st = 2359 - case '0': - st = 2448 - case '1': - st = 2450 - case '2': - st = 2452 - case '3': - st = 2454 - case '4': - st = 2456 - case '5': - st = 2458 - case '6': - st = 2460 - case '7': - st = 2462 - case '8': - st = 2464 - case '9': - st = 2466 - default: - return length - } - - case 2360: - switch b { - case '.': - length = i + 1 - st = 2361 - case '0': - st = 2468 - case '1': - st = 2470 - case '2': - st = 2472 - case '3': - st = 2474 - case '4': - st = 2476 - case '5': - st = 2478 - case '6': - st = 2480 - case '7': - st = 2482 - case '8': - st = 2484 - case '9': - st = 2486 - default: - return length - } - - case 2362: - switch b { - case '.': - length = i + 1 - st = 2363 - case '0': - st = 2488 - case '1': - st = 2490 - case '2': - st = 2492 - case '3': - st = 2494 - case '4': - st = 2496 - case '5': - st = 2498 - case '6': - st = 2500 - case '7': - st = 2502 - case '8': - st = 2504 - case '9': - st = 2506 - default: - return length - } - - case 2364: - switch b { - case '.': - length = i + 1 - st = 2365 - case '0': - st = 2508 - case '1': - st = 2510 - case '2': - st = 2512 - case '3': - st = 2514 - case '4': - st = 2516 - case '5': - st = 2518 - case '6': - st = 2520 - case '7': - st = 2522 - case '8': - st = 2524 - case '9': - st = 2526 - default: - return length - } - - case 2366: - switch b { - case '.': - length = i + 1 - st = 2367 - case '0': - st = 2528 - case '1': - st = 2530 - case '2': - st = 2532 - case '3': - st = 2534 - case '4': - st = 2536 - case '5': - st = 2538 - case '6': - st = 2540 - case '7': - st = 2542 - case '8': - st = 2544 - case '9': - st = 2546 - default: - return length - } - - case 2368: - switch b { - case '.': - length = i + 1 - st = 2369 - case '0': - st = 2548 - case '1': - st = 2550 - case '2': - st = 2552 - case '3': - st = 2554 - case '4': - st = 2556 - case '5': - st = 2558 - case '6': - st = 2560 - case '7': - st = 2562 - case '8': - st = 2564 - case '9': - st = 2566 - default: - return length - } - - case 2370: - switch b { - case '.': - length = i + 1 - st = 2371 - case '0': - st = 2568 - case '1': - st = 2570 - case '2': - st = 2572 - case '3': - st = 2574 - case '4': - st = 2576 - case '5': - st = 2578 - case '6': - st = 2580 - case '7': - st = 2582 - case '8': - st = 2584 - case '9': - st = 2586 - default: - return length - } - - case 2372: - switch b { - case '.': - length = i + 1 - st = 2373 - case '0': - st = 2588 - case '1': - st = 2590 - case '2': - st = 2592 - case '3': - st = 2594 - case '4': - st = 2596 - case '5': - st = 2598 - case '6': - st = 2600 - case '7': - st = 2602 - case '8': - st = 2604 - case '9': - st = 2606 - default: - return length - } - - case 2374: - switch b { - case '.': - length = i + 1 - st = 2375 - case '0': - st = 2608 - case '1': - st = 2610 - case '2': - st = 2612 - case '3': - st = 2614 - case '4': - st = 2616 - case '5': - st = 2618 - case '6': - st = 2620 - case '7': - st = 2622 - case '8': - st = 2624 - case '9': - st = 2626 - default: - return length - } - - case 2376: - switch b { - case '.': - length = i + 1 - st = 2377 - case '0': - st = 2628 - case '1': - st = 2630 - case '2': - st = 2632 - case '3': - st = 2634 - case '4': - st = 2636 - case '5': - st = 2638 - case '6': - st = 2640 - case '7': - st = 2642 - case '8': - st = 2644 - case '9': - st = 2646 - default: - return length - } - - case 2378: - switch b { - case '.': - length = i + 1 - st = 2379 - case '0': - st = 2648 - case '1': - st = 2650 - case '2': - st = 2652 - case '3': - st = 2654 - case '4': - st = 2656 - case '5': - st = 2658 - case '6': - st = 2660 - case '7': - st = 2662 - case '8': - st = 2664 - case '9': - st = 2666 - default: - return length - } - - case 2380: - switch b { - case '.': - length = i + 1 - st = 2381 - case '0': - st = 2668 - case '1': - st = 2670 - case '2': - st = 2672 - case '3': - st = 2674 - case '4': - st = 2676 - case '5': - st = 2678 - case '6': - st = 2680 - case '7': - st = 2682 - case '8': - st = 2684 - case '9': - st = 2686 - default: - return length - } - - case 2382: - switch b { - case '.': - length = i + 1 - st = 2383 - case '0': - st = 2688 - case '1': - st = 2690 - case '2': - st = 2692 - case '3': - st = 2694 - case '4': - st = 2696 - case '5': - st = 2698 - case '6': - st = 2700 - case '7': - st = 2702 - case '8': - st = 2704 - case '9': - st = 2706 - default: - return length - } - - case 2384: - switch b { - case '.': - length = i + 1 - st = 2385 - case '0': - st = 2708 - case '1': - st = 2710 - case '2': - st = 2712 - case '3': - st = 2714 - case '4': - st = 2716 - case '5': - st = 2718 - case '6': - st = 2720 - case '7': - st = 2722 - case '8': - st = 2724 - case '9': - st = 2726 - default: - return length - } - - case 2386: - switch b { - case '.': - length = i + 1 - st = 2387 - case '0': - st = 2728 - case '1': - st = 2730 - case '2': - st = 2732 - case '3': - st = 2734 - case '4': - st = 2736 - case '5': - st = 2738 - case '6': - st = 2740 - case '7': - st = 2742 - case '8': - st = 2744 - case '9': - st = 2746 - default: - return length - } - - case 2388: - switch b { - case '.': - length = i + 1 - st = 2389 - case '0': - st = 2748 - case '1': - st = 2750 - case '2': - st = 2752 - case '3': - st = 2754 - case '4': - st = 2756 - case '5': - st = 2758 - case '6': - st = 2760 - case '7': - st = 2762 - case '8': - st = 2764 - case '9': - st = 2766 - default: - return length - } - - case 2390: - switch b { - case '.': - length = i + 1 - st = 2391 - case '0': - st = 2768 - case '1': - st = 2770 - case '2': - st = 2772 - case '3': - st = 2774 - case '4': - st = 2776 - case '5': - st = 2778 - case '6': - st = 2780 - case '7': - st = 2782 - case '8': - st = 2784 - case '9': - st = 2786 - default: - return length - } - - case 2392: - switch b { - case '.': - length = i + 1 - st = 2393 - case '0': - st = 2788 - case '1': - st = 2790 - case '2': - st = 2792 - case '3': - st = 2794 - case '4': - st = 2796 - case '5': - st = 2798 - case '6': - st = 2800 - case '7': - st = 2802 - case '8': - st = 2804 - case '9': - st = 2806 - default: - return length - } - - case 2394: - switch b { - case '.': - length = i + 1 - st = 2395 - case '0': - st = 2808 - case '1': - st = 2810 - case '2': - st = 2812 - case '3': - st = 2814 - case '4': - st = 2816 - case '5': - st = 2818 - case '6': - st = 2820 - case '7': - st = 2822 - case '8': - st = 2824 - case '9': - st = 2826 - default: - return length - } - - case 2396: - switch b { - case '.': - length = i + 1 - st = 2397 - case '0': - st = 2828 - case '1': - st = 2830 - case '2': - st = 2832 - case '3': - st = 2834 - case '4': - st = 2836 - case '5': - st = 2838 - case '6': - st = 2840 - case '7': - st = 2842 - case '8': - st = 2844 - case '9': - st = 2846 - default: - return length - } - - case 2398: - switch b { - case '.': - length = i + 1 - st = 2399 - case '0': - st = 2848 - case '1': - st = 2850 - case '2': - st = 2852 - case '3': - st = 2854 - case '4': - st = 2856 - case '5': - st = 2858 - default: - return length - } - - case 2400: - switch b { - case '.': - length = i + 1 - st = 2401 - default: - return length - } - - case 2402: - switch b { - case '.': - length = i + 1 - st = 2403 - default: - return length - } - - case 2404: - switch b { - case '.': - length = i + 1 - st = 2405 - default: - return length - } - - case 2406: - switch b { - case '.': - length = i + 1 - st = 2407 - default: - return length - } - - case 2408: - switch b { - case '.': - length = i + 1 - st = 2409 - default: - return length - } - - case 2410: - switch b { - case '.': - length = i + 1 - st = 2411 - default: - return length - } - - case 2412: - switch b { - case '.': - length = i + 1 - st = 2413 - default: - return length - } - - case 2414: - switch b { - case '.': - length = i + 1 - st = 2415 - default: - return length - } - - case 2416: - switch b { - case '.': - length = i + 1 - st = 2417 - default: - return length - } - - case 2418: - switch b { - case '.': - length = i + 1 - st = 2419 - default: - return length - } - - case 2420: - switch b { - case '.': - length = i + 1 - st = 2421 - default: - return length - } - - case 2422: - switch b { - case '.': - length = i + 1 - st = 2423 - default: - return length - } - - case 2424: - switch b { - case '.': - length = i + 1 - st = 2425 - default: - return length - } - - case 2426: - switch b { - case '.': - length = i + 1 - st = 2427 - default: - return length - } - - case 2428: - switch b { - case '.': - length = i + 1 - st = 2429 - default: - return length - } - - case 2430: - switch b { - case '.': - length = i + 1 - st = 2431 - default: - return length - } - - case 2432: - switch b { - case '.': - length = i + 1 - st = 2433 - default: - return length - } - - case 2434: - switch b { - case '.': - length = i + 1 - st = 2435 - default: - return length - } - - case 2436: - switch b { - case '.': - length = i + 1 - st = 2437 - default: - return length - } - - case 2438: - switch b { - case '.': - length = i + 1 - st = 2439 - default: - return length - } - - case 2440: - switch b { - case '.': - length = i + 1 - st = 2441 - default: - return length - } - - case 2442: - switch b { - case '.': - length = i + 1 - st = 2443 - default: - return length - } - - case 2444: - switch b { - case '.': - length = i + 1 - st = 2445 - default: - return length - } - - case 2446: - switch b { - case '.': - length = i + 1 - st = 2447 - default: - return length - } - - case 2448: - switch b { - case '.': - length = i + 1 - st = 2449 - default: - return length - } - - case 2450: - switch b { - case '.': - length = i + 1 - st = 2451 - default: - return length - } - - case 2452: - switch b { - case '.': - length = i + 1 - st = 2453 - default: - return length - } - - case 2454: - switch b { - case '.': - length = i + 1 - st = 2455 - default: - return length - } - - case 2456: - switch b { - case '.': - length = i + 1 - st = 2457 - default: - return length - } - - case 2458: - switch b { - case '.': - length = i + 1 - st = 2459 - default: - return length - } - - case 2460: - switch b { - case '.': - length = i + 1 - st = 2461 - default: - return length - } - - case 2462: - switch b { - case '.': - length = i + 1 - st = 2463 - default: - return length - } - - case 2464: - switch b { - case '.': - length = i + 1 - st = 2465 - default: - return length - } - - case 2466: - switch b { - case '.': - length = i + 1 - st = 2467 - default: - return length - } - - case 2468: - switch b { - case '.': - length = i + 1 - st = 2469 - default: - return length - } - - case 2470: - switch b { - case '.': - length = i + 1 - st = 2471 - default: - return length - } - - case 2472: - switch b { - case '.': - length = i + 1 - st = 2473 - default: - return length - } - - case 2474: - switch b { - case '.': - length = i + 1 - st = 2475 - default: - return length - } - - case 2476: - switch b { - case '.': - length = i + 1 - st = 2477 - default: - return length - } - - case 2478: - switch b { - case '.': - length = i + 1 - st = 2479 - default: - return length - } - - case 2480: - switch b { - case '.': - length = i + 1 - st = 2481 - default: - return length - } - - case 2482: - switch b { - case '.': - length = i + 1 - st = 2483 - default: - return length - } - - case 2484: - switch b { - case '.': - length = i + 1 - st = 2485 - default: - return length - } - - case 2486: - switch b { - case '.': - length = i + 1 - st = 2487 - default: - return length - } - - case 2488: - switch b { - case '.': - length = i + 1 - st = 2489 - default: - return length - } - - case 2490: - switch b { - case '.': - length = i + 1 - st = 2491 - default: - return length - } - - case 2492: - switch b { - case '.': - length = i + 1 - st = 2493 - default: - return length - } - - case 2494: - switch b { - case '.': - length = i + 1 - st = 2495 - default: - return length - } - - case 2496: - switch b { - case '.': - length = i + 1 - st = 2497 - default: - return length - } - - case 2498: - switch b { - case '.': - length = i + 1 - st = 2499 - default: - return length - } - - case 2500: - switch b { - case '.': - length = i + 1 - st = 2501 - default: - return length - } - - case 2502: - switch b { - case '.': - length = i + 1 - st = 2503 - default: - return length - } - - case 2504: - switch b { - case '.': - length = i + 1 - st = 2505 - default: - return length - } - - case 2506: - switch b { - case '.': - length = i + 1 - st = 2507 - default: - return length - } - - case 2508: - switch b { - case '.': - length = i + 1 - st = 2509 - default: - return length - } - - case 2510: - switch b { - case '.': - length = i + 1 - st = 2511 - default: - return length - } - - case 2512: - switch b { - case '.': - length = i + 1 - st = 2513 - default: - return length - } - - case 2514: - switch b { - case '.': - length = i + 1 - st = 2515 - default: - return length - } - - case 2516: - switch b { - case '.': - length = i + 1 - st = 2517 - default: - return length - } - - case 2518: - switch b { - case '.': - length = i + 1 - st = 2519 - default: - return length - } - - case 2520: - switch b { - case '.': - length = i + 1 - st = 2521 - default: - return length - } - - case 2522: - switch b { - case '.': - length = i + 1 - st = 2523 - default: - return length - } - - case 2524: - switch b { - case '.': - length = i + 1 - st = 2525 - default: - return length - } - - case 2526: - switch b { - case '.': - length = i + 1 - st = 2527 - default: - return length - } - - case 2528: - switch b { - case '.': - length = i + 1 - st = 2529 - default: - return length - } - - case 2530: - switch b { - case '.': - length = i + 1 - st = 2531 - default: - return length - } - - case 2532: - switch b { - case '.': - length = i + 1 - st = 2533 - default: - return length - } - - case 2534: - switch b { - case '.': - length = i + 1 - st = 2535 - default: - return length - } - - case 2536: - switch b { - case '.': - length = i + 1 - st = 2537 - default: - return length - } - - case 2538: - switch b { - case '.': - length = i + 1 - st = 2539 - default: - return length - } - - case 2540: - switch b { - case '.': - length = i + 1 - st = 2541 - default: - return length - } - - case 2542: - switch b { - case '.': - length = i + 1 - st = 2543 - default: - return length - } - - case 2544: - switch b { - case '.': - length = i + 1 - st = 2545 - default: - return length - } - - case 2546: - switch b { - case '.': - length = i + 1 - st = 2547 - default: - return length - } - - case 2548: - switch b { - case '.': - length = i + 1 - st = 2549 - default: - return length - } - - case 2550: - switch b { - case '.': - length = i + 1 - st = 2551 - default: - return length - } - - case 2552: - switch b { - case '.': - length = i + 1 - st = 2553 - default: - return length - } - - case 2554: - switch b { - case '.': - length = i + 1 - st = 2555 - default: - return length - } - - case 2556: - switch b { - case '.': - length = i + 1 - st = 2557 - default: - return length - } - - case 2558: - switch b { - case '.': - length = i + 1 - st = 2559 - default: - return length - } - - case 2560: - switch b { - case '.': - length = i + 1 - st = 2561 - default: - return length - } - - case 2562: - switch b { - case '.': - length = i + 1 - st = 2563 - default: - return length - } - - case 2564: - switch b { - case '.': - length = i + 1 - st = 2565 - default: - return length - } - - case 2566: - switch b { - case '.': - length = i + 1 - st = 2567 - default: - return length - } - - case 2568: - switch b { - case '.': - length = i + 1 - st = 2569 - default: - return length - } - - case 2570: - switch b { - case '.': - length = i + 1 - st = 2571 - default: - return length - } - - case 2572: - switch b { - case '.': - length = i + 1 - st = 2573 - default: - return length - } - - case 2574: - switch b { - case '.': - length = i + 1 - st = 2575 - default: - return length - } - - case 2576: - switch b { - case '.': - length = i + 1 - st = 2577 - default: - return length - } - - case 2578: - switch b { - case '.': - length = i + 1 - st = 2579 - default: - return length - } - - case 2580: - switch b { - case '.': - length = i + 1 - st = 2581 - default: - return length - } - - case 2582: - switch b { - case '.': - length = i + 1 - st = 2583 - default: - return length - } - - case 2584: - switch b { - case '.': - length = i + 1 - st = 2585 - default: - return length - } - - case 2586: - switch b { - case '.': - length = i + 1 - st = 2587 - default: - return length - } - - case 2588: - switch b { - case '.': - length = i + 1 - st = 2589 - default: - return length - } - - case 2590: - switch b { - case '.': - length = i + 1 - st = 2591 - default: - return length - } - - case 2592: - switch b { - case '.': - length = i + 1 - st = 2593 - default: - return length - } - - case 2594: - switch b { - case '.': - length = i + 1 - st = 2595 - default: - return length - } - - case 2596: - switch b { - case '.': - length = i + 1 - st = 2597 - default: - return length - } - - case 2598: - switch b { - case '.': - length = i + 1 - st = 2599 - default: - return length - } - - case 2600: - switch b { - case '.': - length = i + 1 - st = 2601 - default: - return length - } - - case 2602: - switch b { - case '.': - length = i + 1 - st = 2603 - default: - return length - } - - case 2604: - switch b { - case '.': - length = i + 1 - st = 2605 - default: - return length - } - - case 2606: - switch b { - case '.': - length = i + 1 - st = 2607 - default: - return length - } - - case 2608: - switch b { - case '.': - length = i + 1 - st = 2609 - default: - return length - } - - case 2610: - switch b { - case '.': - length = i + 1 - st = 2611 - default: - return length - } - - case 2612: - switch b { - case '.': - length = i + 1 - st = 2613 - default: - return length - } - - case 2614: - switch b { - case '.': - length = i + 1 - st = 2615 - default: - return length - } - - case 2616: - switch b { - case '.': - length = i + 1 - st = 2617 - default: - return length - } - - case 2618: - switch b { - case '.': - length = i + 1 - st = 2619 - default: - return length - } - - case 2620: - switch b { - case '.': - length = i + 1 - st = 2621 - default: - return length - } - - case 2622: - switch b { - case '.': - length = i + 1 - st = 2623 - default: - return length - } - - case 2624: - switch b { - case '.': - length = i + 1 - st = 2625 - default: - return length - } - - case 2626: - switch b { - case '.': - length = i + 1 - st = 2627 - default: - return length - } - - case 2628: - switch b { - case '.': - length = i + 1 - st = 2629 - default: - return length - } - - case 2630: - switch b { - case '.': - length = i + 1 - st = 2631 - default: - return length - } - - case 2632: - switch b { - case '.': - length = i + 1 - st = 2633 - default: - return length - } - - case 2634: - switch b { - case '.': - length = i + 1 - st = 2635 - default: - return length - } - - case 2636: - switch b { - case '.': - length = i + 1 - st = 2637 - default: - return length - } - - case 2638: - switch b { - case '.': - length = i + 1 - st = 2639 - default: - return length - } - - case 2640: - switch b { - case '.': - length = i + 1 - st = 2641 - default: - return length - } - - case 2642: - switch b { - case '.': - length = i + 1 - st = 2643 - default: - return length - } - - case 2644: - switch b { - case '.': - length = i + 1 - st = 2645 - default: - return length - } - - case 2646: - switch b { - case '.': - length = i + 1 - st = 2647 - default: - return length - } - - case 2648: - switch b { - case '.': - length = i + 1 - st = 2649 - default: - return length - } - - case 2650: - switch b { - case '.': - length = i + 1 - st = 2651 - default: - return length - } - - case 2652: - switch b { - case '.': - length = i + 1 - st = 2653 - default: - return length - } - - case 2654: - switch b { - case '.': - length = i + 1 - st = 2655 - default: - return length - } - - case 2656: - switch b { - case '.': - length = i + 1 - st = 2657 - default: - return length - } - - case 2658: - switch b { - case '.': - length = i + 1 - st = 2659 - default: - return length - } - - case 2660: - switch b { - case '.': - length = i + 1 - st = 2661 - default: - return length - } - - case 2662: - switch b { - case '.': - length = i + 1 - st = 2663 - default: - return length - } - - case 2664: - switch b { - case '.': - length = i + 1 - st = 2665 - default: - return length - } - - case 2666: - switch b { - case '.': - length = i + 1 - st = 2667 - default: - return length - } - - case 2668: - switch b { - case '.': - length = i + 1 - st = 2669 - default: - return length - } - - case 2670: - switch b { - case '.': - length = i + 1 - st = 2671 - default: - return length - } - - case 2672: - switch b { - case '.': - length = i + 1 - st = 2673 - default: - return length - } - - case 2674: - switch b { - case '.': - length = i + 1 - st = 2675 - default: - return length - } - - case 2676: - switch b { - case '.': - length = i + 1 - st = 2677 - default: - return length - } - - case 2678: - switch b { - case '.': - length = i + 1 - st = 2679 - default: - return length - } - - case 2680: - switch b { - case '.': - length = i + 1 - st = 2681 - default: - return length - } - - case 2682: - switch b { - case '.': - length = i + 1 - st = 2683 - default: - return length - } - - case 2684: - switch b { - case '.': - length = i + 1 - st = 2685 - default: - return length - } - - case 2686: - switch b { - case '.': - length = i + 1 - st = 2687 - default: - return length - } - - case 2688: - switch b { - case '.': - length = i + 1 - st = 2689 - default: - return length - } - - case 2690: - switch b { - case '.': - length = i + 1 - st = 2691 - default: - return length - } - - case 2692: - switch b { - case '.': - length = i + 1 - st = 2693 - default: - return length - } - - case 2694: - switch b { - case '.': - length = i + 1 - st = 2695 - default: - return length - } - - case 2696: - switch b { - case '.': - length = i + 1 - st = 2697 - default: - return length - } - - case 2698: - switch b { - case '.': - length = i + 1 - st = 2699 - default: - return length - } - - case 2700: - switch b { - case '.': - length = i + 1 - st = 2701 - default: - return length - } - - case 2702: - switch b { - case '.': - length = i + 1 - st = 2703 - default: - return length - } - - case 2704: - switch b { - case '.': - length = i + 1 - st = 2705 - default: - return length - } - - case 2706: - switch b { - case '.': - length = i + 1 - st = 2707 - default: - return length - } - - case 2708: - switch b { - case '.': - length = i + 1 - st = 2709 - default: - return length - } - - case 2710: - switch b { - case '.': - length = i + 1 - st = 2711 - default: - return length - } - - case 2712: - switch b { - case '.': - length = i + 1 - st = 2713 - default: - return length - } - - case 2714: - switch b { - case '.': - length = i + 1 - st = 2715 - default: - return length - } - - case 2716: - switch b { - case '.': - length = i + 1 - st = 2717 - default: - return length - } - - case 2718: - switch b { - case '.': - length = i + 1 - st = 2719 - default: - return length - } - - case 2720: - switch b { - case '.': - length = i + 1 - st = 2721 - default: - return length - } - - case 2722: - switch b { - case '.': - length = i + 1 - st = 2723 - default: - return length - } - - case 2724: - switch b { - case '.': - length = i + 1 - st = 2725 - default: - return length - } - - case 2726: - switch b { - case '.': - length = i + 1 - st = 2727 - default: - return length - } - - case 2728: - switch b { - case '.': - length = i + 1 - st = 2729 - default: - return length - } - - case 2730: - switch b { - case '.': - length = i + 1 - st = 2731 - default: - return length - } - - case 2732: - switch b { - case '.': - length = i + 1 - st = 2733 - default: - return length - } - - case 2734: - switch b { - case '.': - length = i + 1 - st = 2735 - default: - return length - } - - case 2736: - switch b { - case '.': - length = i + 1 - st = 2737 - default: - return length - } - - case 2738: - switch b { - case '.': - length = i + 1 - st = 2739 - default: - return length - } - - case 2740: - switch b { - case '.': - length = i + 1 - st = 2741 - default: - return length - } - - case 2742: - switch b { - case '.': - length = i + 1 - st = 2743 - default: - return length - } - - case 2744: - switch b { - case '.': - length = i + 1 - st = 2745 - default: - return length - } - - case 2746: - switch b { - case '.': - length = i + 1 - st = 2747 - default: - return length - } - - case 2748: - switch b { - case '.': - length = i + 1 - st = 2749 - default: - return length - } - - case 2750: - switch b { - case '.': - length = i + 1 - st = 2751 - default: - return length - } - - case 2752: - switch b { - case '.': - length = i + 1 - st = 2753 - default: - return length - } - - case 2754: - switch b { - case '.': - length = i + 1 - st = 2755 - default: - return length - } - - case 2756: - switch b { - case '.': - length = i + 1 - st = 2757 - default: - return length - } - - case 2758: - switch b { - case '.': - length = i + 1 - st = 2759 - default: - return length - } - - case 2760: - switch b { - case '.': - length = i + 1 - st = 2761 - default: - return length - } - - case 2762: - switch b { - case '.': - length = i + 1 - st = 2763 - default: - return length - } - - case 2764: - switch b { - case '.': - length = i + 1 - st = 2765 - default: - return length - } - - case 2766: - switch b { - case '.': - length = i + 1 - st = 2767 - default: - return length - } - - case 2768: - switch b { - case '.': - length = i + 1 - st = 2769 - default: - return length - } - - case 2770: - switch b { - case '.': - length = i + 1 - st = 2771 - default: - return length - } - - case 2772: - switch b { - case '.': - length = i + 1 - st = 2773 - default: - return length - } - - case 2774: - switch b { - case '.': - length = i + 1 - st = 2775 - default: - return length - } - - case 2776: - switch b { - case '.': - length = i + 1 - st = 2777 - default: - return length - } - - case 2778: - switch b { - case '.': - length = i + 1 - st = 2779 - default: - return length - } - - case 2780: - switch b { - case '.': - length = i + 1 - st = 2781 - default: - return length - } - - case 2782: - switch b { - case '.': - length = i + 1 - st = 2783 - default: - return length - } - - case 2784: - switch b { - case '.': - length = i + 1 - st = 2785 - default: - return length - } - - case 2786: - switch b { - case '.': - length = i + 1 - st = 2787 - default: - return length - } - - case 2788: - switch b { - case '.': - length = i + 1 - st = 2789 - default: - return length - } - - case 2790: - switch b { - case '.': - length = i + 1 - st = 2791 - default: - return length - } - - case 2792: - switch b { - case '.': - length = i + 1 - st = 2793 - default: - return length - } - - case 2794: - switch b { - case '.': - length = i + 1 - st = 2795 - default: - return length - } - - case 2796: - switch b { - case '.': - length = i + 1 - st = 2797 - default: - return length - } - - case 2798: - switch b { - case '.': - length = i + 1 - st = 2799 - default: - return length - } - - case 2800: - switch b { - case '.': - length = i + 1 - st = 2801 - default: - return length - } - - case 2802: - switch b { - case '.': - length = i + 1 - st = 2803 - default: - return length - } - - case 2804: - switch b { - case '.': - length = i + 1 - st = 2805 - default: - return length - } - - case 2806: - switch b { - case '.': - length = i + 1 - st = 2807 - default: - return length - } - - case 2808: - switch b { - case '.': - length = i + 1 - st = 2809 - default: - return length - } - - case 2810: - switch b { - case '.': - length = i + 1 - st = 2811 - default: - return length - } - - case 2812: - switch b { - case '.': - length = i + 1 - st = 2813 - default: - return length - } - - case 2814: - switch b { - case '.': - length = i + 1 - st = 2815 - default: - return length - } - - case 2816: - switch b { - case '.': - length = i + 1 - st = 2817 - default: - return length - } - - case 2818: - switch b { - case '.': - length = i + 1 - st = 2819 - default: - return length - } - - case 2820: - switch b { - case '.': - length = i + 1 - st = 2821 - default: - return length - } - - case 2822: - switch b { - case '.': - length = i + 1 - st = 2823 - default: - return length - } - - case 2824: - switch b { - case '.': - length = i + 1 - st = 2825 - default: - return length - } - - case 2826: - switch b { - case '.': - length = i + 1 - st = 2827 - default: - return length - } - - case 2828: - switch b { - case '.': - length = i + 1 - st = 2829 - default: - return length - } - - case 2830: - switch b { - case '.': - length = i + 1 - st = 2831 - default: - return length - } - - case 2832: - switch b { - case '.': - length = i + 1 - st = 2833 - default: - return length - } - - case 2834: - switch b { - case '.': - length = i + 1 - st = 2835 - default: - return length - } - - case 2836: - switch b { - case '.': - length = i + 1 - st = 2837 - default: - return length - } - - case 2838: - switch b { - case '.': - length = i + 1 - st = 2839 - default: - return length - } - - case 2840: - switch b { - case '.': - length = i + 1 - st = 2841 - default: - return length - } - - case 2842: - switch b { - case '.': - length = i + 1 - st = 2843 - default: - return length - } - - case 2844: - switch b { - case '.': - length = i + 1 - st = 2845 - default: - return length - } - - case 2846: - switch b { - case '.': - length = i + 1 - st = 2847 - default: - return length - } - - case 2848: - switch b { - case '.': - length = i + 1 - st = 2849 - default: - return length - } - - case 2850: - switch b { - case '.': - length = i + 1 - st = 2851 - default: - return length - } - - case 2852: - switch b { - case '.': - length = i + 1 - st = 2853 - default: - return length - } - - case 2854: - switch b { - case '.': - length = i + 1 - st = 2855 - default: - return length - } - - case 2856: - switch b { - case '.': - length = i + 1 - st = 2857 - default: - return length - } - - case 2858: - switch b { - case '.': - length = i + 1 - st = 2859 - default: - return length - } - - case 2860: - switch b { - case '-': - st = 2861 - default: - return length - } - - case 2861: - switch b { - case '-': - length = i + 1 - st = 2862 - default: - return length - } - - } - } - - return length -} diff --git a/vendor/github.com/golang-commonmark/markdown/linkify/linkify.go b/vendor/github.com/golang-commonmark/markdown/linkify/linkify.go deleted file mode 100644 index 617f35a1..00000000 --- a/vendor/github.com/golang-commonmark/markdown/linkify/linkify.go +++ /dev/null @@ -1,463 +0,0 @@ -// Copyright 2015 The Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package linkify provides a way to find links in plain text. -package linkify - -import ( - "unicode/utf8" - - "github.com/golang-commonmark/markdown/byteutil" -) - -// Link represents a link found in a string with a schema and a position in the string. -type Link struct { - Scheme string - Start, End int -} - -func max(a, b int) int { - if a >= b { - return a - } - return b -} - -// Links returns links found in s. -func Links(s string) (links []Link) { - for i := 0; i < len(s)-2; i++ { - switch s[i] { - case '.': // IP address or domain name - if i == 0 { - continue // . at the start of a line - } - if length := match(s[i+1:]); length > 0 { - pos := i + 1 + length - switch s[pos-1] { - case '.': // IP address - if pos >= len(s) { - continue // . at the end of line - } - if !byteutil.IsDigit(s[i-1]) { - i = pos - continue // . should be preceded by a digit - } - if !byteutil.IsDigit(s[pos]) { - i = pos - continue // . should be followed by a digit - } - - // find the start of the IP address - j := i - 2 - m := max(0, j-3) - for j >= m && byteutil.IsDigit(s[j]) { - j-- - } - if i-2-j > 2 { - i = pos + 1 - continue // at most 3 digits - } - start := 0 - if j >= 0 { - r, rlen := utf8.DecodeLastRuneInString(s[:j+1]) - if !isPunctOrSpaceOrControl(r) { - i = pos + 1 - continue - } - switch r { - case '.', ':', '/', '\\', '-', '_': - i = pos + 1 - continue - } - start = j + 2 - rlen - } - - length, ok := skipIPv4(s[start:]) - if !ok { - i = pos + 1 - continue - } - end := start + length - if end == len(s) { - links = append(links, Link{ - Scheme: "", - Start: start, - End: end, - }) - return - } - - r, _ := utf8.DecodeRuneInString(s[end:]) - if !isPunctOrSpaceOrControl(r) { - continue - } - - end = skipPort(s, end) - end = skipPath(s, end) - end = skipQuery(s, end) - end = skipFragment(s, end) - end = unskipPunct(s, end) - - if end < len(s) { - r, _ = utf8.DecodeRuneInString(s[end:]) - if !isPunctOrSpaceOrControl(r) || r == '%' { - continue - } - } - - links = append(links, Link{ - Scheme: "", - Start: start, - End: end, - }) - i = end - - default: // domain name - r, _ := utf8.DecodeLastRuneInString(s[:i]) - if !isLetterOrDigit(r) { - continue // should be preceded by a letter or a digit - } - - if pos == len(s) { - start, ok := findHostnameStart(s, i) - if !ok { - continue - } - links = append(links, Link{ - Scheme: "", - Start: start, - End: pos, - }) - return - } - - if s[i+1:pos] != "xn--" { - r, _ = utf8.DecodeRuneInString(s[pos:]) - if isLetterOrDigit(r) { - continue // should not be followed by a letter or a digit - } - } - - end, dot, ok := findHostnameEnd(s, pos) - if !ok { - continue - } - dot = max(dot, i) - - if !(dot+5 <= len(s) && s[dot+1:dot+5] == "xn--") { - if length := match(s[dot+1:]); dot+length+1 != end { - continue - } - } - - start, ok := findHostnameStart(s, i) - if !ok { - continue - } - - end = skipPort(s, end) - end = skipPath(s, end) - end = skipQuery(s, end) - end = skipFragment(s, end) - end = unskipPunct(s, end) - - if end < len(s) { - r, _ = utf8.DecodeRuneInString(s[end:]) - if !isPunctOrSpaceOrControl(r) || r == '%' { - continue // should be followed by punctuation or space - } - } - - links = append(links, Link{ - Scheme: "", - Start: start, - End: end, - }) - i = end - } - } - - case '/': // schema-less link - if s[i+1] != '/' { - continue - } - - if i > 0 { - if s[i-1] == ':' { - i++ - continue // should not be preceded by a colon - } - r, _ := utf8.DecodeLastRuneInString(s[:i]) - if !isPunctOrSpaceOrControl(r) { - i++ - continue // should be preceded by punctuation or space - } - } - - r, _ := utf8.DecodeRuneInString(s[i+2:]) - if !isLetterOrDigit(r) { - i++ - continue // should be followed by a letter or a digit - } - - start := i - end, dot, ok := findHostnameEnd(s, i+2) - if !ok { - continue - } - if s[i+2:end] != "localhost" { - if dot == -1 { - continue // no dot - } - if length, ok := skipIPv4(s[i+2:]); !ok || i+2+length != end { - if length := match(s[dot+1:]); dot+length+1 != end { - continue - } - } - } - - end = skipPort(s, end) - end = skipPath(s, end) - end = skipQuery(s, end) - end = skipFragment(s, end) - end = unskipPunct(s, end) - - if end < len(s) { - r, _ = utf8.DecodeRuneInString(s[end:]) - if !isPunctOrSpaceOrControl(r) || r == '%' { - continue // should be followed by punctuation or space - } - } - - links = append(links, Link{ - Scheme: "//", - Start: start, - End: end, - }) - i = end - - case ':': // http, https, ftp, mailto or localhost - if i < 3 { // at least ftp: - continue - } - - if i >= 9 && s[i-1] == 't' && s[i-9:i] == "localhost" { - j := i - 9 - if !byteutil.IsDigit(s[j+10]) { - continue - } - if j > 0 { - r, _ := utf8.DecodeLastRuneInString(s[:j]) - if !isPunctOrSpaceOrControl(r) { - i++ - continue // should be preceded by punctuation or space - } - } - - start := j - pos := j + 9 - end := skipPort(s, pos) - if end == pos { - continue // invalid port - } - end = skipPath(s, end) - end = skipQuery(s, end) - end = skipFragment(s, end) - end = unskipPunct(s, end) - - if end < len(s) { - r, _ := utf8.DecodeRuneInString(s[end:]) - if !isPunctOrSpaceOrControl(r) || r == '%' { - i++ - continue // should be followed by punctuation or space - } - } - - links = append(links, Link{ - Scheme: "", - Start: start, - End: end, - }) - i = end - - break - } - - j := i - 1 - var start int - var schema string - - switch byteutil.ByteToLower(s[j]) { - case 'o': // mailto - if j < 5 { - continue // too short for mailto - } - if len(s)-j < 8 { - continue // insufficient length after - } - if byteutil.ToLower(s[j-5:j+2]) != "mailto:" { - continue - } - r, _ := utf8.DecodeLastRuneInString(s[:j-5]) - if isLetterOrDigit(r) { - continue // should not be preceded by a letter or a digit - } - r, _ = utf8.DecodeRuneInString(s[j+2:]) - if !isAllowedInEmail(r) { - continue // should be followed by a valid e-mail character - } - - start = j - 5 - end, ok := findEmailEnd(s, j+2) - if !ok { - continue - } - - links = append(links, Link{ - Scheme: "mailto:", - Start: start, - End: end, - }) - i = end - continue // continue processing - - case 'p': // http or ftp - if len(s)-j < 8 { - continue // insufficient length after - } - switch byteutil.ByteToLower(s[j-2]) { - case 'f': - if byteutil.ToLower(s[j-2:j+4]) != "ftp://" { - continue - } - start = j - 2 - schema = "ftp:" - case 't': - if j < 3 { - continue - } - if byteutil.ToLower(s[j-3:j+4]) != "http://" { - continue - } - start = j - 3 - schema = "http:" - default: - continue - } - - case 's': // https - if j < 4 { - continue // too short for https - } - if len(s)-j < 8 { - continue // insufficient length after - } - start = j - 4 - if byteutil.ToLower(s[start:j+4]) != "https://" { - continue - } - schema = "https:" - - default: - continue - } - - // http, https or ftp - - if start > 0 { - r, _ := utf8.DecodeLastRuneInString(s[:start]) - if !isPunctOrSpaceOrControl(r) { - continue // should be preceded by punctuation or space - } - } - - r, _ := utf8.DecodeRuneInString(s[j+4:]) - if !isLetterOrDigit(r) { - continue // should be followed by a letter or a digit - } - - end, dot, ok := findHostnameEnd(s, j+4) - if !ok { - continue - } - if s[j+4:end] != "localhost" { - if dot == -1 { - continue // no dot - } - if length, ok := skipIPv4(s[j+4:]); !ok || j+4+length != end { - if !(dot+5 <= len(s) && s[dot+1:dot+5] == "xn--") { - if length := match(s[dot+1:]); dot+length+1 != end { - continue - } - } - } - } - - end = skipPort(s, end) - end = skipPath(s, end) - end = skipQuery(s, end) - end = skipFragment(s, end) - end = unskipPunct(s, end) - - if end < len(s) { - r, _ = utf8.DecodeRuneInString(s[end:]) - if !isPunctOrSpaceOrControl(r) || r == '%' { - continue // should be followed by punctuation or space - } - } - - links = append(links, Link{ - Scheme: schema, - Start: start, - End: end, - }) - i = end - - case '@': // schema-less e-mail - if i == 0 { - continue // @ at the start of a line - } - - if len(s)-i < 5 { - continue // insufficient length after - } - - r, _ := utf8.DecodeLastRuneInString(s[:i]) - if !isAllowedInEmail(r) { - continue // should be preceded by a valid e-mail character - } - - r, _ = utf8.DecodeRuneInString(s[i+1:]) - if !isLetterOrDigit(r) { - continue // should be followed by a letter or a digit - } - - start, ok := findEmailStart(s, i-1) - if !ok { - continue - } - - end, dot, ok := findHostnameEnd(s, i+1) - if !ok { - continue - } - if dot == -1 { - continue // no dot - } - if !(dot+5 <= len(s) && s[dot+1:dot+5] == "xn--") { - if length := match(s[dot+1:]); dot+length+1 != end { - continue - } - } - - links = append(links, Link{ - Scheme: "mailto:", - Start: start, - End: end, - }) - i = end - } - } - return -} diff --git a/vendor/github.com/golang-commonmark/markdown/linkify/url.go b/vendor/github.com/golang-commonmark/markdown/linkify/url.go deleted file mode 100644 index 78acb506..00000000 --- a/vendor/github.com/golang-commonmark/markdown/linkify/url.go +++ /dev/null @@ -1,412 +0,0 @@ -// Copyright 2015 The Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package linkify - -import ( - "unicode/utf8" - - "github.com/golang-commonmark/markdown/byteutil" -) - -func atoi3(s string, start int) (int, bool) { - n := 0 - var i int - for i = start; i < len(s) && byteutil.IsDigit(s[i]); i++ { - n = n*10 + int(s[i]-'0') - if n > 255 { - return 0, false - } - } - if i == start { - return 0, false - } - return i, true -} - -func skipIPv4(s string) (_ int, _ bool) { - j := 0 - for i := 0; i < 4; i++ { - if j >= len(s) { - return - } - if i > 0 { - if s[j] != '.' { - return - } - j++ - } - if n, ok := atoi3(s, j); !ok { - return - } else { - j = n - } - } - return j, true -} - -func atoi5(s string, start int) (int, bool) { - n := 0 - var i int - for i = start; i < len(s) && byteutil.IsDigit(s[i]); i++ { - n = n*10 + int(s[i]-'0') - if n > 65535 { - return 0, false - } - } - if i == start || n == 0 { - return 0, false - } - return i, true -} - -func skipPort(s string, start int) int { - if start >= len(s) || s[start] != ':' { - return start - } - end, ok := atoi5(s, start+1) - if !ok { - return start - } - return end -} - -func skipPath(s string, start int) int { - if start >= len(s) || s[start] != '/' { - return start // skip empty path - } - var stack []rune - var notClosedIndex int - var nHyphen int - end := start + 1 -loop: - for end < len(s) { - r, rlen := utf8.DecodeRuneInString(s[end:]) - if r == utf8.RuneError { - nHyphen = 0 - break - } - - switch { - case isUnreserved(r): - if r == '-' { - nHyphen++ - if nHyphen > 1 { - break loop - } - } else { - nHyphen = 0 - } - case isSubDelimiter(r) || r == '[' || r == ']': - nHyphen = 0 - switch r { - case '[', '(': - if len(stack) == 0 { - notClosedIndex = end - } - stack = append(stack, r) - case ']', ')': - opening := '[' - if r == ')' { - opening = '(' - } - if len(stack) == 0 || stack[len(stack)-1] != opening { - break loop - } - stack = stack[:len(stack)-1] - } - case r == '/' || r == ':' || r == '@': - nHyphen = 0 - case r == '%': - nHyphen = 0 - if end+2 >= len(s) { - break loop - } - if !(byteutil.IsHexDigit(s[end+1]) && - byteutil.IsHexDigit(s[end+2])) { - break loop - } - end += 2 - default: - nHyphen = 0 - if r != ' ' || len(stack) == 0 { - break loop - } - } - end += rlen - } - if len(stack) > 0 { - return notClosedIndex - } - if nHyphen > 0 { - return end - nHyphen + 1 - } - return end -} - -func skipQuery(s string, start int) int { - if start >= len(s) || s[start] != '?' { - return start - } - var stack []rune - var notClosedIndex int - var nHyphen int - end := start + 1 -loop: - for end < len(s) { - r, rlen := utf8.DecodeRuneInString(s[end:]) - if r == utf8.RuneError { - nHyphen = 0 - break - } - - switch { - case isUnreserved(r): - if r == '-' { - nHyphen++ - if nHyphen > 1 { - break loop - } - } else { - nHyphen = 0 - } - case isSubDelimiter(r) || r == '[' || r == ']': - nHyphen = 0 - switch r { - case '[', '(': - if len(stack) == 0 { - notClosedIndex = end - } - stack = append(stack, r) - case ']', ')': - opening := '[' - if r == ')' { - opening = '(' - } - if len(stack) == 0 || stack[len(stack)-1] != opening { - break loop - } - stack = stack[:len(stack)-1] - } - case r == '?' || r == '/' || r == ':' || r == '@': - nHyphen = 0 - case r == '%': - nHyphen = 0 - if end+2 >= len(s) { - break loop - } - if !(byteutil.IsHexDigit(s[end+1]) && - byteutil.IsHexDigit(s[end+2])) { - break loop - } - end += 2 - default: - nHyphen = 0 - if r != ' ' || len(stack) == 0 { - break loop - } - } - end += rlen - } - if len(stack) > 0 { - return notClosedIndex - } - if nHyphen > 0 { - return end - nHyphen + 1 - } - return end -} - -func skipFragment(s string, start int) int { - if start >= len(s) || s[start] != '#' { - return start - } - var stack []rune - var notClosedIndex int - var nHyphen int - end := start + 1 -loop: - for end < len(s) { - r, rlen := utf8.DecodeRuneInString(s[end:]) - if r == utf8.RuneError { - nHyphen = 0 - break - } - - switch { - case isUnreserved(r): - if r == '-' { - nHyphen++ - if nHyphen > 1 { - break loop - } - } else { - nHyphen = 0 - } - case isSubDelimiter(r) || r == '[' || r == ']': - nHyphen = 0 - switch r { - case '[', '(': - if len(stack) == 0 { - notClosedIndex = end - } - stack = append(stack, r) - case ']', ')': - opening := '[' - if r == ')' { - opening = '(' - } - if len(stack) == 0 || stack[len(stack)-1] != opening { - break loop - } - stack = stack[:len(stack)-1] - } - case r == '?' || r == '/' || r == ':' || r == '@': - nHyphen = 0 - case r == '%': - nHyphen = 0 - if end+2 >= len(s) { - break loop - } - if !(byteutil.IsHexDigit(s[end+1]) && - byteutil.IsHexDigit(s[end+2])) { - break loop - } - end += 2 - default: - nHyphen = 0 - if r != ' ' || len(stack) == 0 { - break loop - } - } - end += rlen - } - if len(stack) > 0 { - return notClosedIndex - } - if nHyphen > 0 { - return end - nHyphen + 1 - } - return end -} - -func unskipPunct(s string, start int) int { - end := start - 1 - if end < 0 || end >= len(s) || !basicPunct[s[end]] { - return start - } - return end -} - -func findHostnameStart(s string, start int) (_ int, _ bool) { - end := start - lastDot := true - nHyphen := 0 -loop: - for end > 0 { - r, rlen := utf8.DecodeLastRuneInString(s[:end]) - if r == utf8.RuneError { - return - } - - switch { - case isLetterOrDigit(r): - lastDot = false - nHyphen = 0 - case r == '.': - if nHyphen > 0 { - return - } - lastDot = true - case r == '-': - if end == start { - return - } - if lastDot { - return - } - nHyphen++ - if nHyphen == 3 { - return - } - case r == ':' || r == '/' || r == '\\' || r == '_': - return - case isPunctOrSpaceOrControl(r): - break loop - default: - return - } - end -= rlen - } - if lastDot || nHyphen > 0 { - return - } - return end, true -} - -func findHostnameEnd(s string, start int) (_ int, _ int, _ bool) { - end := start - lastDot := false - lastDotPos := -1 - nHyphen := 0 -loop: - for end < len(s) { - r, rlen := utf8.DecodeRuneInString(s[end:]) - if r == utf8.RuneError { - return - } - - switch { - case isLetterOrDigit(r): - lastDot = false - nHyphen = 0 - case r == '.': - if nHyphen > 0 { - return - } - if lastDot { - break loop - } - lastDot = true - lastDotPos = end - nHyphen = 0 - case r == '-': - lastDot = false - if end == start { - return - } - if lastDot { - return - } - nHyphen++ - if nHyphen == 3 { - break loop - } - case r == '\\' || r == '_': - return - case isPunctOrSpaceOrControl(r): - break loop - default: - return - } - end += rlen - } - - if nHyphen > 0 { - end -= nHyphen - } else if lastDot { - if s[end-1] == '.' { - end-- - } - lastDotPos = end - 1 - for lastDotPos >= start && s[lastDotPos] != '.' { - lastDotPos-- - } - if lastDotPos < start { - lastDotPos = -1 - } - } - - return end, lastDotPos, true -} diff --git a/vendor/github.com/golang/protobuf/proto/Makefile b/vendor/github.com/golang/protobuf/proto/Makefile deleted file mode 100644 index e2e0651a..00000000 --- a/vendor/github.com/golang/protobuf/proto/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -# Go support for Protocol Buffers - Google's data interchange format -# -# Copyright 2010 The Go Authors. All rights reserved. -# https://github.com/golang/protobuf -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -install: - go install - -test: install generate-test-pbs - go test - - -generate-test-pbs: - make install - make -C testdata - protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto - make diff --git a/vendor/github.com/golang/protobuf/proto/clone.go b/vendor/github.com/golang/protobuf/proto/clone.go index e392575b..3cd3249f 100644 --- a/vendor/github.com/golang/protobuf/proto/clone.go +++ b/vendor/github.com/golang/protobuf/proto/clone.go @@ -35,22 +35,39 @@ package proto import ( + "fmt" "log" "reflect" "strings" ) // Clone returns a deep copy of a protocol buffer. -func Clone(pb Message) Message { - in := reflect.ValueOf(pb) +func Clone(src Message) Message { + in := reflect.ValueOf(src) if in.IsNil() { - return pb + return src } - out := reflect.New(in.Type().Elem()) - // out is empty so a merge is a deep copy. - mergeStruct(out.Elem(), in.Elem()) - return out.Interface().(Message) + dst := out.Interface().(Message) + Merge(dst, src) + return dst +} + +// Merger is the interface representing objects that can merge messages of the same type. +type Merger interface { + // Merge merges src into this message. + // Required and optional fields that are set in src will be set to that value in dst. + // Elements of repeated fields will be appended. + // + // Merge may panic if called with a different argument type than the receiver. + Merge(src Message) +} + +// generatedMerger is the custom merge method that generated protos will have. +// We must add this method since a generate Merge method will conflict with +// many existing protos that have a Merge data field already defined. +type generatedMerger interface { + XXX_Merge(src Message) } // Merge merges src into dst. @@ -58,17 +75,24 @@ func Clone(pb Message) Message { // Elements of repeated fields will be appended. // Merge panics if src and dst are not the same type, or if dst is nil. func Merge(dst, src Message) { + if m, ok := dst.(Merger); ok { + m.Merge(src) + return + } + in := reflect.ValueOf(src) out := reflect.ValueOf(dst) if out.IsNil() { panic("proto: nil destination") } if in.Type() != out.Type() { - // Explicit test prior to mergeStruct so that mistyped nils will fail - panic("proto: type mismatch") + panic(fmt.Sprintf("proto.Merge(%T, %T) type mismatch", dst, src)) } if in.IsNil() { - // Merging nil into non-nil is a quiet no-op + return // Merge from nil src is a noop + } + if m, ok := dst.(generatedMerger); ok { + m.XXX_Merge(src) return } mergeStruct(out.Elem(), in.Elem()) @@ -84,7 +108,7 @@ func mergeStruct(out, in reflect.Value) { mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i]) } - if emIn, ok := extendable(in.Addr().Interface()); ok { + if emIn, err := extendable(in.Addr().Interface()); err == nil { emOut, _ := extendable(out.Addr().Interface()) mIn, muIn := emIn.extensionsRead() if mIn != nil { diff --git a/vendor/github.com/golang/protobuf/proto/decode.go b/vendor/github.com/golang/protobuf/proto/decode.go index aa207298..d9aa3c42 100644 --- a/vendor/github.com/golang/protobuf/proto/decode.go +++ b/vendor/github.com/golang/protobuf/proto/decode.go @@ -39,8 +39,6 @@ import ( "errors" "fmt" "io" - "os" - "reflect" ) // errOverflow is returned when an integer is too large to be represented. @@ -50,10 +48,6 @@ var errOverflow = errors.New("proto: integer overflow") // wire type is encountered. It does not get returned to user code. var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof") -// The fundamental decoders that interpret bytes on the wire. -// Those that take integer types all return uint64 and are -// therefore of type valueDecoder. - // DecodeVarint reads a varint-encoded integer from the slice. // It returns the integer and the number of bytes consumed, or // zero if there is not enough. @@ -267,9 +261,6 @@ func (p *Buffer) DecodeZigzag32() (x uint64, err error) { return } -// These are not ValueDecoders: they produce an array of bytes or a string. -// bytes, embedded messages - // DecodeRawBytes reads a count-delimited byte buffer from the Buffer. // This is the format used for the bytes protocol buffer // type and for embedded messages. @@ -311,81 +302,29 @@ func (p *Buffer) DecodeStringBytes() (s string, err error) { return string(buf), nil } -// Skip the next item in the buffer. Its wire type is decoded and presented as an argument. -// If the protocol buffer has extensions, and the field matches, add it as an extension. -// Otherwise, if the XXX_unrecognized field exists, append the skipped data there. -func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error { - oi := o.index - - err := o.skip(t, tag, wire) - if err != nil { - return err - } - - if !unrecField.IsValid() { - return nil - } - - ptr := structPointer_Bytes(base, unrecField) - - // Add the skipped field to struct field - obuf := o.buf - - o.buf = *ptr - o.EncodeVarint(uint64(tag<<3 | wire)) - *ptr = append(o.buf, obuf[oi:o.index]...) - - o.buf = obuf - - return nil -} - -// Skip the next item in the buffer. Its wire type is decoded and presented as an argument. -func (o *Buffer) skip(t reflect.Type, tag, wire int) error { - - var u uint64 - var err error - - switch wire { - case WireVarint: - _, err = o.DecodeVarint() - case WireFixed64: - _, err = o.DecodeFixed64() - case WireBytes: - _, err = o.DecodeRawBytes(false) - case WireFixed32: - _, err = o.DecodeFixed32() - case WireStartGroup: - for { - u, err = o.DecodeVarint() - if err != nil { - break - } - fwire := int(u & 0x7) - if fwire == WireEndGroup { - break - } - ftag := int(u >> 3) - err = o.skip(t, ftag, fwire) - if err != nil { - break - } - } - default: - err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t) - } - return err -} - // Unmarshaler is the interface representing objects that can -// unmarshal themselves. The method should reset the receiver before -// decoding starts. The argument points to data that may be +// unmarshal themselves. The argument points to data that may be // overwritten, so implementations should not keep references to the // buffer. +// Unmarshal implementations should not clear the receiver. +// Any unmarshaled data should be merged into the receiver. +// Callers of Unmarshal that do not want to retain existing data +// should Reset the receiver before calling Unmarshal. type Unmarshaler interface { Unmarshal([]byte) error } +// newUnmarshaler is the interface representing objects that can +// unmarshal themselves. The semantics are identical to Unmarshaler. +// +// This exists to support protoc-gen-go generated messages. +// The proto package will stop type-asserting to this interface in the future. +// +// DO NOT DEPEND ON THIS. +type newUnmarshaler interface { + XXX_Unmarshal([]byte) error +} + // Unmarshal parses the protocol buffer representation in buf and places the // decoded result in pb. If the struct underlying pb does not match // the data in buf, the results can be unpredictable. @@ -395,7 +334,13 @@ type Unmarshaler interface { // to preserve and append to existing data. func Unmarshal(buf []byte, pb Message) error { pb.Reset() - return UnmarshalMerge(buf, pb) + if u, ok := pb.(newUnmarshaler); ok { + return u.XXX_Unmarshal(buf) + } + if u, ok := pb.(Unmarshaler); ok { + return u.Unmarshal(buf) + } + return NewBuffer(buf).Unmarshal(pb) } // UnmarshalMerge parses the protocol buffer representation in buf and @@ -405,8 +350,16 @@ func Unmarshal(buf []byte, pb Message) error { // UnmarshalMerge merges into existing data in pb. // Most code should use Unmarshal instead. func UnmarshalMerge(buf []byte, pb Message) error { - // If the object can unmarshal itself, let it. + if u, ok := pb.(newUnmarshaler); ok { + return u.XXX_Unmarshal(buf) + } if u, ok := pb.(Unmarshaler); ok { + // NOTE: The history of proto have unfortunately been inconsistent + // whether Unmarshaler should or should not implicitly clear itself. + // Some implementations do, most do not. + // Thus, calling this here may or may not do what people want. + // + // See https://github.com/golang/protobuf/issues/424 return u.Unmarshal(buf) } return NewBuffer(buf).Unmarshal(pb) @@ -422,12 +375,17 @@ func (p *Buffer) DecodeMessage(pb Message) error { } // DecodeGroup reads a tag-delimited group from the Buffer. +// StartGroup tag is already consumed. This function consumes +// EndGroup tag. func (p *Buffer) DecodeGroup(pb Message) error { - typ, base, err := getbase(pb) - if err != nil { - return err + b := p.buf[p.index:] + x, y := findEndGroup(b) + if x < 0 { + return io.ErrUnexpectedEOF } - return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base) + err := Unmarshal(b[:x], pb) + p.index += y + return err } // Unmarshal parses the protocol buffer representation in the @@ -438,533 +396,33 @@ func (p *Buffer) DecodeGroup(pb Message) error { // Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal. func (p *Buffer) Unmarshal(pb Message) error { // If the object can unmarshal itself, let it. + if u, ok := pb.(newUnmarshaler); ok { + err := u.XXX_Unmarshal(p.buf[p.index:]) + p.index = len(p.buf) + return err + } if u, ok := pb.(Unmarshaler); ok { + // NOTE: The history of proto have unfortunately been inconsistent + // whether Unmarshaler should or should not implicitly clear itself. + // Some implementations do, most do not. + // Thus, calling this here may or may not do what people want. + // + // See https://github.com/golang/protobuf/issues/424 err := u.Unmarshal(p.buf[p.index:]) p.index = len(p.buf) return err } - typ, base, err := getbase(pb) - if err != nil { - return err - } - - err = p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), false, base) - - if collectStats { - stats.Decode++ - } - - return err -} - -// unmarshalType does the work of unmarshaling a structure. -func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group bool, base structPointer) error { - var state errorState - required, reqFields := prop.reqCount, uint64(0) - - var err error - for err == nil && o.index < len(o.buf) { - oi := o.index - var u uint64 - u, err = o.DecodeVarint() - if err != nil { - break - } - wire := int(u & 0x7) - if wire == WireEndGroup { - if is_group { - if required > 0 { - // Not enough information to determine the exact field. - // (See below.) - return &RequiredNotSetError{"{Unknown}"} - } - return nil // input is satisfied - } - return fmt.Errorf("proto: %s: wiretype end group for non-group", st) - } - tag := int(u >> 3) - if tag <= 0 { - return fmt.Errorf("proto: %s: illegal tag %d (wire type %d)", st, tag, wire) - } - fieldnum, ok := prop.decoderTags.get(tag) - if !ok { - // Maybe it's an extension? - if prop.extendable { - if e, _ := extendable(structPointer_Interface(base, st)); isExtensionField(e, int32(tag)) { - if err = o.skip(st, tag, wire); err == nil { - extmap := e.extensionsWrite() - ext := extmap[int32(tag)] // may be missing - ext.enc = append(ext.enc, o.buf[oi:o.index]...) - extmap[int32(tag)] = ext - } - continue - } - } - // Maybe it's a oneof? - if prop.oneofUnmarshaler != nil { - m := structPointer_Interface(base, st).(Message) - // First return value indicates whether tag is a oneof field. - ok, err = prop.oneofUnmarshaler(m, tag, wire, o) - if err == ErrInternalBadWireType { - // Map the error to something more descriptive. - // Do the formatting here to save generated code space. - err = fmt.Errorf("bad wiretype for oneof field in %T", m) - } - if ok { - continue - } - } - err = o.skipAndSave(st, tag, wire, base, prop.unrecField) - continue - } - p := prop.Prop[fieldnum] - - if p.dec == nil { - fmt.Fprintf(os.Stderr, "proto: no protobuf decoder for %s.%s\n", st, st.Field(fieldnum).Name) - continue - } - dec := p.dec - if wire != WireStartGroup && wire != p.WireType { - if wire == WireBytes && p.packedDec != nil { - // a packable field - dec = p.packedDec - } else { - err = fmt.Errorf("proto: bad wiretype for field %s.%s: got wiretype %d, want %d", st, st.Field(fieldnum).Name, wire, p.WireType) - continue - } - } - decErr := dec(o, p, base) - if decErr != nil && !state.shouldContinue(decErr, p) { - err = decErr - } - if err == nil && p.Required { - // Successfully decoded a required field. - if tag <= 64 { - // use bitmap for fields 1-64 to catch field reuse. - var mask uint64 = 1 << uint64(tag-1) - if reqFields&mask == 0 { - // new required field - reqFields |= mask - required-- - } - } else { - // This is imprecise. It can be fooled by a required field - // with a tag > 64 that is encoded twice; that's very rare. - // A fully correct implementation would require allocating - // a data structure, which we would like to avoid. - required-- - } - } - } - if err == nil { - if is_group { - return io.ErrUnexpectedEOF - } - if state.err != nil { - return state.err - } - if required > 0 { - // Not enough information to determine the exact field. If we use extra - // CPU, we could determine the field only if the missing required field - // has a tag <= 64 and we check reqFields. - return &RequiredNotSetError{"{Unknown}"} - } - } - return err -} - -// Individual type decoders -// For each, -// u is the decoded value, -// v is a pointer to the field (pointer) in the struct - -// Sizes of the pools to allocate inside the Buffer. -// The goal is modest amortization and allocation -// on at least 16-byte boundaries. -const ( - boolPoolSize = 16 - uint32PoolSize = 8 - uint64PoolSize = 4 -) - -// Decode a bool. -func (o *Buffer) dec_bool(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - if len(o.bools) == 0 { - o.bools = make([]bool, boolPoolSize) - } - o.bools[0] = u != 0 - *structPointer_Bool(base, p.field) = &o.bools[0] - o.bools = o.bools[1:] - return nil -} - -func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - *structPointer_BoolVal(base, p.field) = u != 0 - return nil -} - -// Decode an int32. -func (o *Buffer) dec_int32(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - word32_Set(structPointer_Word32(base, p.field), o, uint32(u)) - return nil -} - -func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u)) - return nil -} - -// Decode an int64. -func (o *Buffer) dec_int64(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - word64_Set(structPointer_Word64(base, p.field), o, u) - return nil -} - -func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - word64Val_Set(structPointer_Word64Val(base, p.field), o, u) - return nil -} - -// Decode a string. -func (o *Buffer) dec_string(p *Properties, base structPointer) error { - s, err := o.DecodeStringBytes() - if err != nil { - return err - } - *structPointer_String(base, p.field) = &s - return nil -} - -func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error { - s, err := o.DecodeStringBytes() - if err != nil { - return err - } - *structPointer_StringVal(base, p.field) = s - return nil -} - -// Decode a slice of bytes ([]byte). -func (o *Buffer) dec_slice_byte(p *Properties, base structPointer) error { - b, err := o.DecodeRawBytes(true) - if err != nil { - return err - } - *structPointer_Bytes(base, p.field) = b - return nil -} - -// Decode a slice of bools ([]bool). -func (o *Buffer) dec_slice_bool(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - v := structPointer_BoolSlice(base, p.field) - *v = append(*v, u != 0) - return nil -} - -// Decode a slice of bools ([]bool) in packed format. -func (o *Buffer) dec_slice_packed_bool(p *Properties, base structPointer) error { - v := structPointer_BoolSlice(base, p.field) - - nn, err := o.DecodeVarint() - if err != nil { - return err - } - nb := int(nn) // number of bytes of encoded bools - fin := o.index + nb - if fin < o.index { - return errOverflow - } - - y := *v - for o.index < fin { - u, err := p.valDec(o) - if err != nil { - return err - } - y = append(y, u != 0) - } - - *v = y - return nil -} - -// Decode a slice of int32s ([]int32). -func (o *Buffer) dec_slice_int32(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - structPointer_Word32Slice(base, p.field).Append(uint32(u)) - return nil -} - -// Decode a slice of int32s ([]int32) in packed format. -func (o *Buffer) dec_slice_packed_int32(p *Properties, base structPointer) error { - v := structPointer_Word32Slice(base, p.field) - - nn, err := o.DecodeVarint() - if err != nil { - return err - } - nb := int(nn) // number of bytes of encoded int32s - - fin := o.index + nb - if fin < o.index { - return errOverflow - } - for o.index < fin { - u, err := p.valDec(o) - if err != nil { - return err - } - v.Append(uint32(u)) - } - return nil -} - -// Decode a slice of int64s ([]int64). -func (o *Buffer) dec_slice_int64(p *Properties, base structPointer) error { - u, err := p.valDec(o) - if err != nil { - return err - } - - structPointer_Word64Slice(base, p.field).Append(u) - return nil -} - -// Decode a slice of int64s ([]int64) in packed format. -func (o *Buffer) dec_slice_packed_int64(p *Properties, base structPointer) error { - v := structPointer_Word64Slice(base, p.field) - - nn, err := o.DecodeVarint() - if err != nil { - return err - } - nb := int(nn) // number of bytes of encoded int64s - - fin := o.index + nb - if fin < o.index { - return errOverflow - } - for o.index < fin { - u, err := p.valDec(o) - if err != nil { - return err - } - v.Append(u) - } - return nil -} - -// Decode a slice of strings ([]string). -func (o *Buffer) dec_slice_string(p *Properties, base structPointer) error { - s, err := o.DecodeStringBytes() - if err != nil { - return err - } - v := structPointer_StringSlice(base, p.field) - *v = append(*v, s) - return nil -} - -// Decode a slice of slice of bytes ([][]byte). -func (o *Buffer) dec_slice_slice_byte(p *Properties, base structPointer) error { - b, err := o.DecodeRawBytes(true) - if err != nil { - return err - } - v := structPointer_BytesSlice(base, p.field) - *v = append(*v, b) - return nil -} - -// Decode a map field. -func (o *Buffer) dec_new_map(p *Properties, base structPointer) error { - raw, err := o.DecodeRawBytes(false) - if err != nil { - return err - } - oi := o.index // index at the end of this map entry - o.index -= len(raw) // move buffer back to start of map entry - - mptr := structPointer_NewAt(base, p.field, p.mtype) // *map[K]V - if mptr.Elem().IsNil() { - mptr.Elem().Set(reflect.MakeMap(mptr.Type().Elem())) - } - v := mptr.Elem() // map[K]V - - // Prepare addressable doubly-indirect placeholders for the key and value types. - // See enc_new_map for why. - keyptr := reflect.New(reflect.PtrTo(p.mtype.Key())).Elem() // addressable *K - keybase := toStructPointer(keyptr.Addr()) // **K - - var valbase structPointer - var valptr reflect.Value - switch p.mtype.Elem().Kind() { - case reflect.Slice: - // []byte - var dummy []byte - valptr = reflect.ValueOf(&dummy) // *[]byte - valbase = toStructPointer(valptr) // *[]byte - case reflect.Ptr: - // message; valptr is **Msg; need to allocate the intermediate pointer - valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V - valptr.Set(reflect.New(valptr.Type().Elem())) - valbase = toStructPointer(valptr) - default: - // everything else - valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V - valbase = toStructPointer(valptr.Addr()) // **V - } - - // Decode. - // This parses a restricted wire format, namely the encoding of a message - // with two fields. See enc_new_map for the format. - for o.index < oi { - // tagcode for key and value properties are always a single byte - // because they have tags 1 and 2. - tagcode := o.buf[o.index] - o.index++ - switch tagcode { - case p.mkeyprop.tagcode[0]: - if err := p.mkeyprop.dec(o, p.mkeyprop, keybase); err != nil { - return err - } - case p.mvalprop.tagcode[0]: - if err := p.mvalprop.dec(o, p.mvalprop, valbase); err != nil { - return err - } - default: - // TODO: Should we silently skip this instead? - return fmt.Errorf("proto: bad map data tag %d", raw[0]) - } - } - keyelem, valelem := keyptr.Elem(), valptr.Elem() - if !keyelem.IsValid() { - keyelem = reflect.Zero(p.mtype.Key()) - } - if !valelem.IsValid() { - valelem = reflect.Zero(p.mtype.Elem()) - } - - v.SetMapIndex(keyelem, valelem) - return nil -} - -// Decode a group. -func (o *Buffer) dec_struct_group(p *Properties, base structPointer) error { - bas := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(bas) { - // allocate new nested message - bas = toStructPointer(reflect.New(p.stype)) - structPointer_SetStructPointer(base, p.field, bas) - } - return o.unmarshalType(p.stype, p.sprop, true, bas) -} - -// Decode an embedded message. -func (o *Buffer) dec_struct_message(p *Properties, base structPointer) (err error) { - raw, e := o.DecodeRawBytes(false) - if e != nil { - return e - } - - bas := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(bas) { - // allocate new nested message - bas = toStructPointer(reflect.New(p.stype)) - structPointer_SetStructPointer(base, p.field, bas) - } - - // If the object can unmarshal itself, let it. - if p.isUnmarshaler { - iv := structPointer_Interface(bas, p.stype) - return iv.(Unmarshaler).Unmarshal(raw) - } - - obuf := o.buf - oi := o.index - o.buf = raw - o.index = 0 - - err = o.unmarshalType(p.stype, p.sprop, false, bas) - o.buf = obuf - o.index = oi - - return err -} - -// Decode a slice of embedded messages. -func (o *Buffer) dec_slice_struct_message(p *Properties, base structPointer) error { - return o.dec_slice_struct(p, false, base) -} - -// Decode a slice of embedded groups. -func (o *Buffer) dec_slice_struct_group(p *Properties, base structPointer) error { - return o.dec_slice_struct(p, true, base) -} - -// Decode a slice of structs ([]*struct). -func (o *Buffer) dec_slice_struct(p *Properties, is_group bool, base structPointer) error { - v := reflect.New(p.stype) - bas := toStructPointer(v) - structPointer_StructPointerSlice(base, p.field).Append(bas) - - if is_group { - err := o.unmarshalType(p.stype, p.sprop, is_group, bas) - return err - } - - raw, err := o.DecodeRawBytes(false) - if err != nil { - return err - } - - // If the object can unmarshal itself, let it. - if p.isUnmarshaler { - iv := v.Interface() - return iv.(Unmarshaler).Unmarshal(raw) - } - - obuf := o.buf - oi := o.index - o.buf = raw - o.index = 0 - - err = o.unmarshalType(p.stype, p.sprop, is_group, bas) - - o.buf = obuf - o.index = oi - + // Slow workaround for messages that aren't Unmarshalers. + // This includes some hand-coded .pb.go files and + // bootstrap protos. + // TODO: fix all of those and then add Unmarshal to + // the Message interface. Then: + // The cast above and code below can be deleted. + // The old unmarshaler can be deleted. + // Clients can call Unmarshal directly (can already do that, actually). + var info InternalMessageInfo + err := info.Unmarshal(pb, p.buf[p.index:]) + p.index = len(p.buf) return err } diff --git a/vendor/github.com/golang/protobuf/proto/discard.go b/vendor/github.com/golang/protobuf/proto/discard.go new file mode 100644 index 00000000..dea2617c --- /dev/null +++ b/vendor/github.com/golang/protobuf/proto/discard.go @@ -0,0 +1,350 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2017 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" +) + +type generatedDiscarder interface { + XXX_DiscardUnknown() +} + +// DiscardUnknown recursively discards all unknown fields from this message +// and all embedded messages. +// +// When unmarshaling a message with unrecognized fields, the tags and values +// of such fields are preserved in the Message. This allows a later call to +// marshal to be able to produce a message that continues to have those +// unrecognized fields. To avoid this, DiscardUnknown is used to +// explicitly clear the unknown fields after unmarshaling. +// +// For proto2 messages, the unknown fields of message extensions are only +// discarded from messages that have been accessed via GetExtension. +func DiscardUnknown(m Message) { + if m, ok := m.(generatedDiscarder); ok { + m.XXX_DiscardUnknown() + return + } + // TODO: Dynamically populate a InternalMessageInfo for legacy messages, + // but the master branch has no implementation for InternalMessageInfo, + // so it would be more work to replicate that approach. + discardLegacy(m) +} + +// DiscardUnknown recursively discards all unknown fields. +func (a *InternalMessageInfo) DiscardUnknown(m Message) { + di := atomicLoadDiscardInfo(&a.discard) + if di == nil { + di = getDiscardInfo(reflect.TypeOf(m).Elem()) + atomicStoreDiscardInfo(&a.discard, di) + } + di.discard(toPointer(&m)) +} + +type discardInfo struct { + typ reflect.Type + + initialized int32 // 0: only typ is valid, 1: everything is valid + lock sync.Mutex + + fields []discardFieldInfo + unrecognized field +} + +type discardFieldInfo struct { + field field // Offset of field, guaranteed to be valid + discard func(src pointer) +} + +var ( + discardInfoMap = map[reflect.Type]*discardInfo{} + discardInfoLock sync.Mutex +) + +func getDiscardInfo(t reflect.Type) *discardInfo { + discardInfoLock.Lock() + defer discardInfoLock.Unlock() + di := discardInfoMap[t] + if di == nil { + di = &discardInfo{typ: t} + discardInfoMap[t] = di + } + return di +} + +func (di *discardInfo) discard(src pointer) { + if src.isNil() { + return // Nothing to do. + } + + if atomic.LoadInt32(&di.initialized) == 0 { + di.computeDiscardInfo() + } + + for _, fi := range di.fields { + sfp := src.offset(fi.field) + fi.discard(sfp) + } + + // For proto2 messages, only discard unknown fields in message extensions + // that have been accessed via GetExtension. + if em, err := extendable(src.asPointerTo(di.typ).Interface()); err == nil { + // Ignore lock since DiscardUnknown is not concurrency safe. + emm, _ := em.extensionsRead() + for _, mx := range emm { + if m, ok := mx.value.(Message); ok { + DiscardUnknown(m) + } + } + } + + if di.unrecognized.IsValid() { + *src.offset(di.unrecognized).toBytes() = nil + } +} + +func (di *discardInfo) computeDiscardInfo() { + di.lock.Lock() + defer di.lock.Unlock() + if di.initialized != 0 { + return + } + t := di.typ + n := t.NumField() + + for i := 0; i < n; i++ { + f := t.Field(i) + if strings.HasPrefix(f.Name, "XXX_") { + continue + } + + dfi := discardFieldInfo{field: toField(&f)} + tf := f.Type + + // Unwrap tf to get its most basic type. + var isPointer, isSlice bool + if tf.Kind() == reflect.Slice && tf.Elem().Kind() != reflect.Uint8 { + isSlice = true + tf = tf.Elem() + } + if tf.Kind() == reflect.Ptr { + isPointer = true + tf = tf.Elem() + } + if isPointer && isSlice && tf.Kind() != reflect.Struct { + panic(fmt.Sprintf("%v.%s cannot be a slice of pointers to primitive types", t, f.Name)) + } + + switch tf.Kind() { + case reflect.Struct: + switch { + case !isPointer: + panic(fmt.Sprintf("%v.%s cannot be a direct struct value", t, f.Name)) + case isSlice: // E.g., []*pb.T + di := getDiscardInfo(tf) + dfi.discard = func(src pointer) { + sps := src.getPointerSlice() + for _, sp := range sps { + if !sp.isNil() { + di.discard(sp) + } + } + } + default: // E.g., *pb.T + di := getDiscardInfo(tf) + dfi.discard = func(src pointer) { + sp := src.getPointer() + if !sp.isNil() { + di.discard(sp) + } + } + } + case reflect.Map: + switch { + case isPointer || isSlice: + panic(fmt.Sprintf("%v.%s cannot be a pointer to a map or a slice of map values", t, f.Name)) + default: // E.g., map[K]V + if tf.Elem().Kind() == reflect.Ptr { // Proto struct (e.g., *T) + dfi.discard = func(src pointer) { + sm := src.asPointerTo(tf).Elem() + if sm.Len() == 0 { + return + } + for _, key := range sm.MapKeys() { + val := sm.MapIndex(key) + DiscardUnknown(val.Interface().(Message)) + } + } + } else { + dfi.discard = func(pointer) {} // Noop + } + } + case reflect.Interface: + // Must be oneof field. + switch { + case isPointer || isSlice: + panic(fmt.Sprintf("%v.%s cannot be a pointer to a interface or a slice of interface values", t, f.Name)) + default: // E.g., interface{} + // TODO: Make this faster? + dfi.discard = func(src pointer) { + su := src.asPointerTo(tf).Elem() + if !su.IsNil() { + sv := su.Elem().Elem().Field(0) + if sv.Kind() == reflect.Ptr && sv.IsNil() { + return + } + switch sv.Type().Kind() { + case reflect.Ptr: // Proto struct (e.g., *T) + DiscardUnknown(sv.Interface().(Message)) + } + } + } + } + default: + continue + } + di.fields = append(di.fields, dfi) + } + + di.unrecognized = invalidField + if f, ok := t.FieldByName("XXX_unrecognized"); ok { + if f.Type != reflect.TypeOf([]byte{}) { + panic("expected XXX_unrecognized to be of type []byte") + } + di.unrecognized = toField(&f) + } + + atomic.StoreInt32(&di.initialized, 1) +} + +func discardLegacy(m Message) { + v := reflect.ValueOf(m) + if v.Kind() != reflect.Ptr || v.IsNil() { + return + } + v = v.Elem() + if v.Kind() != reflect.Struct { + return + } + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + f := t.Field(i) + if strings.HasPrefix(f.Name, "XXX_") { + continue + } + vf := v.Field(i) + tf := f.Type + + // Unwrap tf to get its most basic type. + var isPointer, isSlice bool + if tf.Kind() == reflect.Slice && tf.Elem().Kind() != reflect.Uint8 { + isSlice = true + tf = tf.Elem() + } + if tf.Kind() == reflect.Ptr { + isPointer = true + tf = tf.Elem() + } + if isPointer && isSlice && tf.Kind() != reflect.Struct { + panic(fmt.Sprintf("%T.%s cannot be a slice of pointers to primitive types", m, f.Name)) + } + + switch tf.Kind() { + case reflect.Struct: + switch { + case !isPointer: + panic(fmt.Sprintf("%T.%s cannot be a direct struct value", m, f.Name)) + case isSlice: // E.g., []*pb.T + for j := 0; j < vf.Len(); j++ { + discardLegacy(vf.Index(j).Interface().(Message)) + } + default: // E.g., *pb.T + discardLegacy(vf.Interface().(Message)) + } + case reflect.Map: + switch { + case isPointer || isSlice: + panic(fmt.Sprintf("%T.%s cannot be a pointer to a map or a slice of map values", m, f.Name)) + default: // E.g., map[K]V + tv := vf.Type().Elem() + if tv.Kind() == reflect.Ptr && tv.Implements(protoMessageType) { // Proto struct (e.g., *T) + for _, key := range vf.MapKeys() { + val := vf.MapIndex(key) + discardLegacy(val.Interface().(Message)) + } + } + } + case reflect.Interface: + // Must be oneof field. + switch { + case isPointer || isSlice: + panic(fmt.Sprintf("%T.%s cannot be a pointer to a interface or a slice of interface values", m, f.Name)) + default: // E.g., test_proto.isCommunique_Union interface + if !vf.IsNil() && f.Tag.Get("protobuf_oneof") != "" { + vf = vf.Elem() // E.g., *test_proto.Communique_Msg + if !vf.IsNil() { + vf = vf.Elem() // E.g., test_proto.Communique_Msg + vf = vf.Field(0) // E.g., Proto struct (e.g., *T) or primitive value + if vf.Kind() == reflect.Ptr { + discardLegacy(vf.Interface().(Message)) + } + } + } + } + } + } + + if vf := v.FieldByName("XXX_unrecognized"); vf.IsValid() { + if vf.Type() != reflect.TypeOf([]byte{}) { + panic("expected XXX_unrecognized to be of type []byte") + } + vf.Set(reflect.ValueOf([]byte(nil))) + } + + // For proto2 messages, only discard unknown fields in message extensions + // that have been accessed via GetExtension. + if em, err := extendable(m); err == nil { + // Ignore lock since discardLegacy is not concurrency safe. + emm, _ := em.extensionsRead() + for _, mx := range emm { + if m, ok := mx.value.(Message); ok { + discardLegacy(m) + } + } + } +} diff --git a/vendor/github.com/golang/protobuf/proto/encode.go b/vendor/github.com/golang/protobuf/proto/encode.go index 2b30f846..c27d35f8 100644 --- a/vendor/github.com/golang/protobuf/proto/encode.go +++ b/vendor/github.com/golang/protobuf/proto/encode.go @@ -39,7 +39,6 @@ import ( "errors" "fmt" "reflect" - "sort" ) // RequiredNotSetError is the error returned if Marshal is called with @@ -82,10 +81,6 @@ var ( const maxVarintBytes = 10 // maximum length of a varint -// maxMarshalSize is the largest allowed size of an encoded protobuf, -// since C++ and Java use signed int32s for the size. -const maxMarshalSize = 1<<31 - 1 - // EncodeVarint returns the varint encoding of x. // This is the format for the // int32, int64, uint32, uint64, bool, and enum @@ -119,18 +114,27 @@ func (p *Buffer) EncodeVarint(x uint64) error { // SizeVarint returns the varint encoding size of an integer. func SizeVarint(x uint64) int { - return sizeVarint(x) -} - -func sizeVarint(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } + switch { + case x < 1<<7: + return 1 + case x < 1<<14: + return 2 + case x < 1<<21: + return 3 + case x < 1<<28: + return 4 + case x < 1<<35: + return 5 + case x < 1<<42: + return 6 + case x < 1<<49: + return 7 + case x < 1<<56: + return 8 + case x < 1<<63: + return 9 } - return n + return 10 } // EncodeFixed64 writes a 64-bit integer to the Buffer. @@ -149,10 +153,6 @@ func (p *Buffer) EncodeFixed64(x uint64) error { return nil } -func sizeFixed64(x uint64) int { - return 8 -} - // EncodeFixed32 writes a 32-bit integer to the Buffer. // This is the format for the // fixed32, sfixed32, and float protocol buffer types. @@ -165,10 +165,6 @@ func (p *Buffer) EncodeFixed32(x uint64) error { return nil } -func sizeFixed32(x uint64) int { - return 4 -} - // EncodeZigzag64 writes a zigzag-encoded 64-bit integer // to the Buffer. // This is the format used for the sint64 protocol buffer type. @@ -177,10 +173,6 @@ func (p *Buffer) EncodeZigzag64(x uint64) error { return p.EncodeVarint(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func sizeZigzag64(x uint64) int { - return sizeVarint(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} - // EncodeZigzag32 writes a zigzag-encoded 32-bit integer // to the Buffer. // This is the format used for the sint32 protocol buffer type. @@ -189,10 +181,6 @@ func (p *Buffer) EncodeZigzag32(x uint64) error { return p.EncodeVarint(uint64((uint32(x) << 1) ^ uint32((int32(x) >> 31)))) } -func sizeZigzag32(x uint64) int { - return sizeVarint(uint64((uint32(x) << 1) ^ uint32((int32(x) >> 31)))) -} - // EncodeRawBytes writes a count-delimited byte buffer to the Buffer. // This is the format used for the bytes protocol buffer // type and for embedded messages. @@ -202,11 +190,6 @@ func (p *Buffer) EncodeRawBytes(b []byte) error { return nil } -func sizeRawBytes(b []byte) int { - return sizeVarint(uint64(len(b))) + - len(b) -} - // EncodeStringBytes writes an encoded string to the Buffer. // This is the format used for the proto2 string type. func (p *Buffer) EncodeStringBytes(s string) error { @@ -215,319 +198,17 @@ func (p *Buffer) EncodeStringBytes(s string) error { return nil } -func sizeStringBytes(s string) int { - return sizeVarint(uint64(len(s))) + - len(s) -} - // Marshaler is the interface representing objects that can marshal themselves. type Marshaler interface { Marshal() ([]byte, error) } -// Marshal takes the protocol buffer -// and encodes it into the wire format, returning the data. -func Marshal(pb Message) ([]byte, error) { - // Can the object marshal itself? - if m, ok := pb.(Marshaler); ok { - return m.Marshal() - } - p := NewBuffer(nil) - err := p.Marshal(pb) - if p.buf == nil && err == nil { - // Return a non-nil slice on success. - return []byte{}, nil - } - return p.buf, err -} - // EncodeMessage writes the protocol buffer to the Buffer, // prefixed by a varint-encoded length. func (p *Buffer) EncodeMessage(pb Message) error { - t, base, err := getbase(pb) - if structPointer_IsNil(base) { - return ErrNil - } - if err == nil { - var state errorState - err = p.enc_len_struct(GetProperties(t.Elem()), base, &state) - } - return err -} - -// Marshal takes the protocol buffer -// and encodes it into the wire format, writing the result to the -// Buffer. -func (p *Buffer) Marshal(pb Message) error { - // Can the object marshal itself? - if m, ok := pb.(Marshaler); ok { - data, err := m.Marshal() - p.buf = append(p.buf, data...) - return err - } - - t, base, err := getbase(pb) - if structPointer_IsNil(base) { - return ErrNil - } - if err == nil { - err = p.enc_struct(GetProperties(t.Elem()), base) - } - - if collectStats { - (stats).Encode++ // Parens are to work around a goimports bug. - } - - if len(p.buf) > maxMarshalSize { - return ErrTooLarge - } - return err -} - -// Size returns the encoded size of a protocol buffer. -func Size(pb Message) (n int) { - // Can the object marshal itself? If so, Size is slow. - // TODO: add Size to Marshaler, or add a Sizer interface. - if m, ok := pb.(Marshaler); ok { - b, _ := m.Marshal() - return len(b) - } - - t, base, err := getbase(pb) - if structPointer_IsNil(base) { - return 0 - } - if err == nil { - n = size_struct(GetProperties(t.Elem()), base) - } - - if collectStats { - (stats).Size++ // Parens are to work around a goimports bug. - } - - return -} - -// Individual type encoders. - -// Encode a bool. -func (o *Buffer) enc_bool(p *Properties, base structPointer) error { - v := *structPointer_Bool(base, p.field) - if v == nil { - return ErrNil - } - x := 0 - if *v { - x = 1 - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func (o *Buffer) enc_proto3_bool(p *Properties, base structPointer) error { - v := *structPointer_BoolVal(base, p.field) - if !v { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, 1) - return nil -} - -func size_bool(p *Properties, base structPointer) int { - v := *structPointer_Bool(base, p.field) - if v == nil { - return 0 - } - return len(p.tagcode) + 1 // each bool takes exactly one byte -} - -func size_proto3_bool(p *Properties, base structPointer) int { - v := *structPointer_BoolVal(base, p.field) - if !v && !p.oneof { - return 0 - } - return len(p.tagcode) + 1 // each bool takes exactly one byte -} - -// Encode an int32. -func (o *Buffer) enc_int32(p *Properties, base structPointer) error { - v := structPointer_Word32(base, p.field) - if word32_IsNil(v) { - return ErrNil - } - x := int32(word32_Get(v)) // permit sign extension to use full 64-bit range - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func (o *Buffer) enc_proto3_int32(p *Properties, base structPointer) error { - v := structPointer_Word32Val(base, p.field) - x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range - if x == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func size_int32(p *Properties, base structPointer) (n int) { - v := structPointer_Word32(base, p.field) - if word32_IsNil(v) { - return 0 - } - x := int32(word32_Get(v)) // permit sign extension to use full 64-bit range - n += len(p.tagcode) - n += p.valSize(uint64(x)) - return -} - -func size_proto3_int32(p *Properties, base structPointer) (n int) { - v := structPointer_Word32Val(base, p.field) - x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range - if x == 0 && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += p.valSize(uint64(x)) - return -} - -// Encode a uint32. -// Exactly the same as int32, except for no sign extension. -func (o *Buffer) enc_uint32(p *Properties, base structPointer) error { - v := structPointer_Word32(base, p.field) - if word32_IsNil(v) { - return ErrNil - } - x := word32_Get(v) - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func (o *Buffer) enc_proto3_uint32(p *Properties, base structPointer) error { - v := structPointer_Word32Val(base, p.field) - x := word32Val_Get(v) - if x == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, uint64(x)) - return nil -} - -func size_uint32(p *Properties, base structPointer) (n int) { - v := structPointer_Word32(base, p.field) - if word32_IsNil(v) { - return 0 - } - x := word32_Get(v) - n += len(p.tagcode) - n += p.valSize(uint64(x)) - return -} - -func size_proto3_uint32(p *Properties, base structPointer) (n int) { - v := structPointer_Word32Val(base, p.field) - x := word32Val_Get(v) - if x == 0 && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += p.valSize(uint64(x)) - return -} - -// Encode an int64. -func (o *Buffer) enc_int64(p *Properties, base structPointer) error { - v := structPointer_Word64(base, p.field) - if word64_IsNil(v) { - return ErrNil - } - x := word64_Get(v) - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, x) - return nil -} - -func (o *Buffer) enc_proto3_int64(p *Properties, base structPointer) error { - v := structPointer_Word64Val(base, p.field) - x := word64Val_Get(v) - if x == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, x) - return nil -} - -func size_int64(p *Properties, base structPointer) (n int) { - v := structPointer_Word64(base, p.field) - if word64_IsNil(v) { - return 0 - } - x := word64_Get(v) - n += len(p.tagcode) - n += p.valSize(x) - return -} - -func size_proto3_int64(p *Properties, base structPointer) (n int) { - v := structPointer_Word64Val(base, p.field) - x := word64Val_Get(v) - if x == 0 && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += p.valSize(x) - return -} - -// Encode a string. -func (o *Buffer) enc_string(p *Properties, base structPointer) error { - v := *structPointer_String(base, p.field) - if v == nil { - return ErrNil - } - x := *v - o.buf = append(o.buf, p.tagcode...) - o.EncodeStringBytes(x) - return nil -} - -func (o *Buffer) enc_proto3_string(p *Properties, base structPointer) error { - v := *structPointer_StringVal(base, p.field) - if v == "" { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeStringBytes(v) - return nil -} - -func size_string(p *Properties, base structPointer) (n int) { - v := *structPointer_String(base, p.field) - if v == nil { - return 0 - } - x := *v - n += len(p.tagcode) - n += sizeStringBytes(x) - return -} - -func size_proto3_string(p *Properties, base structPointer) (n int) { - v := *structPointer_StringVal(base, p.field) - if v == "" && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += sizeStringBytes(v) - return + siz := Size(pb) + p.EncodeVarint(uint64(siz)) + return p.Marshal(pb) } // All protocol buffer fields are nillable, but be careful. @@ -538,825 +219,3 @@ func isNil(v reflect.Value) bool { } return false } - -// Encode a message struct. -func (o *Buffer) enc_struct_message(p *Properties, base structPointer) error { - var state errorState - structp := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(structp) { - return ErrNil - } - - // Can the object marshal itself? - if p.isMarshaler { - m := structPointer_Interface(structp, p.stype).(Marshaler) - data, err := m.Marshal() - if err != nil && !state.shouldContinue(err, nil) { - return err - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(data) - return state.err - } - - o.buf = append(o.buf, p.tagcode...) - return o.enc_len_struct(p.sprop, structp, &state) -} - -func size_struct_message(p *Properties, base structPointer) int { - structp := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(structp) { - return 0 - } - - // Can the object marshal itself? - if p.isMarshaler { - m := structPointer_Interface(structp, p.stype).(Marshaler) - data, _ := m.Marshal() - n0 := len(p.tagcode) - n1 := sizeRawBytes(data) - return n0 + n1 - } - - n0 := len(p.tagcode) - n1 := size_struct(p.sprop, structp) - n2 := sizeVarint(uint64(n1)) // size of encoded length - return n0 + n1 + n2 -} - -// Encode a group struct. -func (o *Buffer) enc_struct_group(p *Properties, base structPointer) error { - var state errorState - b := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(b) { - return ErrNil - } - - o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup)) - err := o.enc_struct(p.sprop, b) - if err != nil && !state.shouldContinue(err, nil) { - return err - } - o.EncodeVarint(uint64((p.Tag << 3) | WireEndGroup)) - return state.err -} - -func size_struct_group(p *Properties, base structPointer) (n int) { - b := structPointer_GetStructPointer(base, p.field) - if structPointer_IsNil(b) { - return 0 - } - - n += sizeVarint(uint64((p.Tag << 3) | WireStartGroup)) - n += size_struct(p.sprop, b) - n += sizeVarint(uint64((p.Tag << 3) | WireEndGroup)) - return -} - -// Encode a slice of bools ([]bool). -func (o *Buffer) enc_slice_bool(p *Properties, base structPointer) error { - s := *structPointer_BoolSlice(base, p.field) - l := len(s) - if l == 0 { - return ErrNil - } - for _, x := range s { - o.buf = append(o.buf, p.tagcode...) - v := uint64(0) - if x { - v = 1 - } - p.valEnc(o, v) - } - return nil -} - -func size_slice_bool(p *Properties, base structPointer) int { - s := *structPointer_BoolSlice(base, p.field) - l := len(s) - if l == 0 { - return 0 - } - return l * (len(p.tagcode) + 1) // each bool takes exactly one byte -} - -// Encode a slice of bools ([]bool) in packed format. -func (o *Buffer) enc_slice_packed_bool(p *Properties, base structPointer) error { - s := *structPointer_BoolSlice(base, p.field) - l := len(s) - if l == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeVarint(uint64(l)) // each bool takes exactly one byte - for _, x := range s { - v := uint64(0) - if x { - v = 1 - } - p.valEnc(o, v) - } - return nil -} - -func size_slice_packed_bool(p *Properties, base structPointer) (n int) { - s := *structPointer_BoolSlice(base, p.field) - l := len(s) - if l == 0 { - return 0 - } - n += len(p.tagcode) - n += sizeVarint(uint64(l)) - n += l // each bool takes exactly one byte - return -} - -// Encode a slice of bytes ([]byte). -func (o *Buffer) enc_slice_byte(p *Properties, base structPointer) error { - s := *structPointer_Bytes(base, p.field) - if s == nil { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(s) - return nil -} - -func (o *Buffer) enc_proto3_slice_byte(p *Properties, base structPointer) error { - s := *structPointer_Bytes(base, p.field) - if len(s) == 0 { - return ErrNil - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(s) - return nil -} - -func size_slice_byte(p *Properties, base structPointer) (n int) { - s := *structPointer_Bytes(base, p.field) - if s == nil && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += sizeRawBytes(s) - return -} - -func size_proto3_slice_byte(p *Properties, base structPointer) (n int) { - s := *structPointer_Bytes(base, p.field) - if len(s) == 0 && !p.oneof { - return 0 - } - n += len(p.tagcode) - n += sizeRawBytes(s) - return -} - -// Encode a slice of int32s ([]int32). -func (o *Buffer) enc_slice_int32(p *Properties, base structPointer) error { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - x := int32(s.Index(i)) // permit sign extension to use full 64-bit range - p.valEnc(o, uint64(x)) - } - return nil -} - -func size_slice_int32(p *Properties, base structPointer) (n int) { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - for i := 0; i < l; i++ { - n += len(p.tagcode) - x := int32(s.Index(i)) // permit sign extension to use full 64-bit range - n += p.valSize(uint64(x)) - } - return -} - -// Encode a slice of int32s ([]int32) in packed format. -func (o *Buffer) enc_slice_packed_int32(p *Properties, base structPointer) error { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - // TODO: Reuse a Buffer. - buf := NewBuffer(nil) - for i := 0; i < l; i++ { - x := int32(s.Index(i)) // permit sign extension to use full 64-bit range - p.valEnc(buf, uint64(x)) - } - - o.buf = append(o.buf, p.tagcode...) - o.EncodeVarint(uint64(len(buf.buf))) - o.buf = append(o.buf, buf.buf...) - return nil -} - -func size_slice_packed_int32(p *Properties, base structPointer) (n int) { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - var bufSize int - for i := 0; i < l; i++ { - x := int32(s.Index(i)) // permit sign extension to use full 64-bit range - bufSize += p.valSize(uint64(x)) - } - - n += len(p.tagcode) - n += sizeVarint(uint64(bufSize)) - n += bufSize - return -} - -// Encode a slice of uint32s ([]uint32). -// Exactly the same as int32, except for no sign extension. -func (o *Buffer) enc_slice_uint32(p *Properties, base structPointer) error { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - x := s.Index(i) - p.valEnc(o, uint64(x)) - } - return nil -} - -func size_slice_uint32(p *Properties, base structPointer) (n int) { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - for i := 0; i < l; i++ { - n += len(p.tagcode) - x := s.Index(i) - n += p.valSize(uint64(x)) - } - return -} - -// Encode a slice of uint32s ([]uint32) in packed format. -// Exactly the same as int32, except for no sign extension. -func (o *Buffer) enc_slice_packed_uint32(p *Properties, base structPointer) error { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - // TODO: Reuse a Buffer. - buf := NewBuffer(nil) - for i := 0; i < l; i++ { - p.valEnc(buf, uint64(s.Index(i))) - } - - o.buf = append(o.buf, p.tagcode...) - o.EncodeVarint(uint64(len(buf.buf))) - o.buf = append(o.buf, buf.buf...) - return nil -} - -func size_slice_packed_uint32(p *Properties, base structPointer) (n int) { - s := structPointer_Word32Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - var bufSize int - for i := 0; i < l; i++ { - bufSize += p.valSize(uint64(s.Index(i))) - } - - n += len(p.tagcode) - n += sizeVarint(uint64(bufSize)) - n += bufSize - return -} - -// Encode a slice of int64s ([]int64). -func (o *Buffer) enc_slice_int64(p *Properties, base structPointer) error { - s := structPointer_Word64Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - p.valEnc(o, s.Index(i)) - } - return nil -} - -func size_slice_int64(p *Properties, base structPointer) (n int) { - s := structPointer_Word64Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - for i := 0; i < l; i++ { - n += len(p.tagcode) - n += p.valSize(s.Index(i)) - } - return -} - -// Encode a slice of int64s ([]int64) in packed format. -func (o *Buffer) enc_slice_packed_int64(p *Properties, base structPointer) error { - s := structPointer_Word64Slice(base, p.field) - l := s.Len() - if l == 0 { - return ErrNil - } - // TODO: Reuse a Buffer. - buf := NewBuffer(nil) - for i := 0; i < l; i++ { - p.valEnc(buf, s.Index(i)) - } - - o.buf = append(o.buf, p.tagcode...) - o.EncodeVarint(uint64(len(buf.buf))) - o.buf = append(o.buf, buf.buf...) - return nil -} - -func size_slice_packed_int64(p *Properties, base structPointer) (n int) { - s := structPointer_Word64Slice(base, p.field) - l := s.Len() - if l == 0 { - return 0 - } - var bufSize int - for i := 0; i < l; i++ { - bufSize += p.valSize(s.Index(i)) - } - - n += len(p.tagcode) - n += sizeVarint(uint64(bufSize)) - n += bufSize - return -} - -// Encode a slice of slice of bytes ([][]byte). -func (o *Buffer) enc_slice_slice_byte(p *Properties, base structPointer) error { - ss := *structPointer_BytesSlice(base, p.field) - l := len(ss) - if l == 0 { - return ErrNil - } - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(ss[i]) - } - return nil -} - -func size_slice_slice_byte(p *Properties, base structPointer) (n int) { - ss := *structPointer_BytesSlice(base, p.field) - l := len(ss) - if l == 0 { - return 0 - } - n += l * len(p.tagcode) - for i := 0; i < l; i++ { - n += sizeRawBytes(ss[i]) - } - return -} - -// Encode a slice of strings ([]string). -func (o *Buffer) enc_slice_string(p *Properties, base structPointer) error { - ss := *structPointer_StringSlice(base, p.field) - l := len(ss) - for i := 0; i < l; i++ { - o.buf = append(o.buf, p.tagcode...) - o.EncodeStringBytes(ss[i]) - } - return nil -} - -func size_slice_string(p *Properties, base structPointer) (n int) { - ss := *structPointer_StringSlice(base, p.field) - l := len(ss) - n += l * len(p.tagcode) - for i := 0; i < l; i++ { - n += sizeStringBytes(ss[i]) - } - return -} - -// Encode a slice of message structs ([]*struct). -func (o *Buffer) enc_slice_struct_message(p *Properties, base structPointer) error { - var state errorState - s := structPointer_StructPointerSlice(base, p.field) - l := s.Len() - - for i := 0; i < l; i++ { - structp := s.Index(i) - if structPointer_IsNil(structp) { - return errRepeatedHasNil - } - - // Can the object marshal itself? - if p.isMarshaler { - m := structPointer_Interface(structp, p.stype).(Marshaler) - data, err := m.Marshal() - if err != nil && !state.shouldContinue(err, nil) { - return err - } - o.buf = append(o.buf, p.tagcode...) - o.EncodeRawBytes(data) - continue - } - - o.buf = append(o.buf, p.tagcode...) - err := o.enc_len_struct(p.sprop, structp, &state) - if err != nil && !state.shouldContinue(err, nil) { - if err == ErrNil { - return errRepeatedHasNil - } - return err - } - } - return state.err -} - -func size_slice_struct_message(p *Properties, base structPointer) (n int) { - s := structPointer_StructPointerSlice(base, p.field) - l := s.Len() - n += l * len(p.tagcode) - for i := 0; i < l; i++ { - structp := s.Index(i) - if structPointer_IsNil(structp) { - return // return the size up to this point - } - - // Can the object marshal itself? - if p.isMarshaler { - m := structPointer_Interface(structp, p.stype).(Marshaler) - data, _ := m.Marshal() - n += sizeRawBytes(data) - continue - } - - n0 := size_struct(p.sprop, structp) - n1 := sizeVarint(uint64(n0)) // size of encoded length - n += n0 + n1 - } - return -} - -// Encode a slice of group structs ([]*struct). -func (o *Buffer) enc_slice_struct_group(p *Properties, base structPointer) error { - var state errorState - s := structPointer_StructPointerSlice(base, p.field) - l := s.Len() - - for i := 0; i < l; i++ { - b := s.Index(i) - if structPointer_IsNil(b) { - return errRepeatedHasNil - } - - o.EncodeVarint(uint64((p.Tag << 3) | WireStartGroup)) - - err := o.enc_struct(p.sprop, b) - - if err != nil && !state.shouldContinue(err, nil) { - if err == ErrNil { - return errRepeatedHasNil - } - return err - } - - o.EncodeVarint(uint64((p.Tag << 3) | WireEndGroup)) - } - return state.err -} - -func size_slice_struct_group(p *Properties, base structPointer) (n int) { - s := structPointer_StructPointerSlice(base, p.field) - l := s.Len() - - n += l * sizeVarint(uint64((p.Tag<<3)|WireStartGroup)) - n += l * sizeVarint(uint64((p.Tag<<3)|WireEndGroup)) - for i := 0; i < l; i++ { - b := s.Index(i) - if structPointer_IsNil(b) { - return // return size up to this point - } - - n += size_struct(p.sprop, b) - } - return -} - -// Encode an extension map. -func (o *Buffer) enc_map(p *Properties, base structPointer) error { - exts := structPointer_ExtMap(base, p.field) - if err := encodeExtensionsMap(*exts); err != nil { - return err - } - - return o.enc_map_body(*exts) -} - -func (o *Buffer) enc_exts(p *Properties, base structPointer) error { - exts := structPointer_Extensions(base, p.field) - - v, mu := exts.extensionsRead() - if v == nil { - return nil - } - - mu.Lock() - defer mu.Unlock() - if err := encodeExtensionsMap(v); err != nil { - return err - } - - return o.enc_map_body(v) -} - -func (o *Buffer) enc_map_body(v map[int32]Extension) error { - // Fast-path for common cases: zero or one extensions. - if len(v) <= 1 { - for _, e := range v { - o.buf = append(o.buf, e.enc...) - } - return nil - } - - // Sort keys to provide a deterministic encoding. - keys := make([]int, 0, len(v)) - for k := range v { - keys = append(keys, int(k)) - } - sort.Ints(keys) - - for _, k := range keys { - o.buf = append(o.buf, v[int32(k)].enc...) - } - return nil -} - -func size_map(p *Properties, base structPointer) int { - v := structPointer_ExtMap(base, p.field) - return extensionsMapSize(*v) -} - -func size_exts(p *Properties, base structPointer) int { - v := structPointer_Extensions(base, p.field) - return extensionsSize(v) -} - -// Encode a map field. -func (o *Buffer) enc_new_map(p *Properties, base structPointer) error { - var state errorState // XXX: or do we need to plumb this through? - - /* - A map defined as - map map_field = N; - is encoded in the same way as - message MapFieldEntry { - key_type key = 1; - value_type value = 2; - } - repeated MapFieldEntry map_field = N; - */ - - v := structPointer_NewAt(base, p.field, p.mtype).Elem() // map[K]V - if v.Len() == 0 { - return nil - } - - keycopy, valcopy, keybase, valbase := mapEncodeScratch(p.mtype) - - enc := func() error { - if err := p.mkeyprop.enc(o, p.mkeyprop, keybase); err != nil { - return err - } - if err := p.mvalprop.enc(o, p.mvalprop, valbase); err != nil && err != ErrNil { - return err - } - return nil - } - - // Don't sort map keys. It is not required by the spec, and C++ doesn't do it. - for _, key := range v.MapKeys() { - val := v.MapIndex(key) - - keycopy.Set(key) - valcopy.Set(val) - - o.buf = append(o.buf, p.tagcode...) - if err := o.enc_len_thing(enc, &state); err != nil { - return err - } - } - return nil -} - -func size_new_map(p *Properties, base structPointer) int { - v := structPointer_NewAt(base, p.field, p.mtype).Elem() // map[K]V - - keycopy, valcopy, keybase, valbase := mapEncodeScratch(p.mtype) - - n := 0 - for _, key := range v.MapKeys() { - val := v.MapIndex(key) - keycopy.Set(key) - valcopy.Set(val) - - // Tag codes for key and val are the responsibility of the sub-sizer. - keysize := p.mkeyprop.size(p.mkeyprop, keybase) - valsize := p.mvalprop.size(p.mvalprop, valbase) - entry := keysize + valsize - // Add on tag code and length of map entry itself. - n += len(p.tagcode) + sizeVarint(uint64(entry)) + entry - } - return n -} - -// mapEncodeScratch returns a new reflect.Value matching the map's value type, -// and a structPointer suitable for passing to an encoder or sizer. -func mapEncodeScratch(mapType reflect.Type) (keycopy, valcopy reflect.Value, keybase, valbase structPointer) { - // Prepare addressable doubly-indirect placeholders for the key and value types. - // This is needed because the element-type encoders expect **T, but the map iteration produces T. - - keycopy = reflect.New(mapType.Key()).Elem() // addressable K - keyptr := reflect.New(reflect.PtrTo(keycopy.Type())).Elem() // addressable *K - keyptr.Set(keycopy.Addr()) // - keybase = toStructPointer(keyptr.Addr()) // **K - - // Value types are more varied and require special handling. - switch mapType.Elem().Kind() { - case reflect.Slice: - // []byte - var dummy []byte - valcopy = reflect.ValueOf(&dummy).Elem() // addressable []byte - valbase = toStructPointer(valcopy.Addr()) - case reflect.Ptr: - // message; the generated field type is map[K]*Msg (so V is *Msg), - // so we only need one level of indirection. - valcopy = reflect.New(mapType.Elem()).Elem() // addressable V - valbase = toStructPointer(valcopy.Addr()) - default: - // everything else - valcopy = reflect.New(mapType.Elem()).Elem() // addressable V - valptr := reflect.New(reflect.PtrTo(valcopy.Type())).Elem() // addressable *V - valptr.Set(valcopy.Addr()) // - valbase = toStructPointer(valptr.Addr()) // **V - } - return -} - -// Encode a struct. -func (o *Buffer) enc_struct(prop *StructProperties, base structPointer) error { - var state errorState - // Encode fields in tag order so that decoders may use optimizations - // that depend on the ordering. - // https://developers.google.com/protocol-buffers/docs/encoding#order - for _, i := range prop.order { - p := prop.Prop[i] - if p.enc != nil { - err := p.enc(o, p, base) - if err != nil { - if err == ErrNil { - if p.Required && state.err == nil { - state.err = &RequiredNotSetError{p.Name} - } - } else if err == errRepeatedHasNil { - // Give more context to nil values in repeated fields. - return errors.New("repeated field " + p.OrigName + " has nil element") - } else if !state.shouldContinue(err, p) { - return err - } - } - if len(o.buf) > maxMarshalSize { - return ErrTooLarge - } - } - } - - // Do oneof fields. - if prop.oneofMarshaler != nil { - m := structPointer_Interface(base, prop.stype).(Message) - if err := prop.oneofMarshaler(m, o); err == ErrNil { - return errOneofHasNil - } else if err != nil { - return err - } - } - - // Add unrecognized fields at the end. - if prop.unrecField.IsValid() { - v := *structPointer_Bytes(base, prop.unrecField) - if len(o.buf)+len(v) > maxMarshalSize { - return ErrTooLarge - } - if len(v) > 0 { - o.buf = append(o.buf, v...) - } - } - - return state.err -} - -func size_struct(prop *StructProperties, base structPointer) (n int) { - for _, i := range prop.order { - p := prop.Prop[i] - if p.size != nil { - n += p.size(p, base) - } - } - - // Add unrecognized fields at the end. - if prop.unrecField.IsValid() { - v := *structPointer_Bytes(base, prop.unrecField) - n += len(v) - } - - // Factor in any oneof fields. - if prop.oneofSizer != nil { - m := structPointer_Interface(base, prop.stype).(Message) - n += prop.oneofSizer(m) - } - - return -} - -var zeroes [20]byte // longer than any conceivable sizeVarint - -// Encode a struct, preceded by its encoded length (as a varint). -func (o *Buffer) enc_len_struct(prop *StructProperties, base structPointer, state *errorState) error { - return o.enc_len_thing(func() error { return o.enc_struct(prop, base) }, state) -} - -// Encode something, preceded by its encoded length (as a varint). -func (o *Buffer) enc_len_thing(enc func() error, state *errorState) error { - iLen := len(o.buf) - o.buf = append(o.buf, 0, 0, 0, 0) // reserve four bytes for length - iMsg := len(o.buf) - err := enc() - if err != nil && !state.shouldContinue(err, nil) { - return err - } - lMsg := len(o.buf) - iMsg - lLen := sizeVarint(uint64(lMsg)) - switch x := lLen - (iMsg - iLen); { - case x > 0: // actual length is x bytes larger than the space we reserved - // Move msg x bytes right. - o.buf = append(o.buf, zeroes[:x]...) - copy(o.buf[iMsg+x:], o.buf[iMsg:iMsg+lMsg]) - case x < 0: // actual length is x bytes smaller than the space we reserved - // Move msg x bytes left. - copy(o.buf[iMsg+x:], o.buf[iMsg:iMsg+lMsg]) - o.buf = o.buf[:len(o.buf)+x] // x is negative - } - // Encode the length in the reserved space. - o.buf = o.buf[:iLen] - o.EncodeVarint(uint64(lMsg)) - o.buf = o.buf[:len(o.buf)+lMsg] - return state.err -} - -// errorState maintains the first error that occurs and updates that error -// with additional context. -type errorState struct { - err error -} - -// shouldContinue reports whether encoding should continue upon encountering the -// given error. If the error is RequiredNotSetError, shouldContinue returns true -// and, if this is the first appearance of that error, remembers it for future -// reporting. -// -// If prop is not nil, it may update any error with additional context about the -// field with the error. -func (s *errorState) shouldContinue(err error, prop *Properties) bool { - // Ignore unset required fields. - reqNotSet, ok := err.(*RequiredNotSetError) - if !ok { - return false - } - if s.err == nil { - if prop != nil { - err = &RequiredNotSetError{prop.Name + "." + reqNotSet.field} - } - s.err = err - } - return true -} diff --git a/vendor/github.com/golang/protobuf/proto/equal.go b/vendor/github.com/golang/protobuf/proto/equal.go index 2ed1cf59..d4db5a1c 100644 --- a/vendor/github.com/golang/protobuf/proto/equal.go +++ b/vendor/github.com/golang/protobuf/proto/equal.go @@ -109,15 +109,6 @@ func equalStruct(v1, v2 reflect.Value) bool { // set/unset mismatch return false } - b1, ok := f1.Interface().(raw) - if ok { - b2 := f2.Interface().(raw) - // RawMessage - if !bytes.Equal(b1.Bytes(), b2.Bytes()) { - return false - } - continue - } f1, f2 = f1.Elem(), f2.Elem() } if !equalAny(f1, f2, sprop.Prop[i]) { @@ -146,11 +137,7 @@ func equalStruct(v1, v2 reflect.Value) bool { u1 := uf.Bytes() u2 := v2.FieldByName("XXX_unrecognized").Bytes() - if !bytes.Equal(u1, u2) { - return false - } - - return true + return bytes.Equal(u1, u2) } // v1 and v2 are known to have the same type. @@ -261,6 +248,15 @@ func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool { m1, m2 := e1.value, e2.value + if m1 == nil && m2 == nil { + // Both have only encoded form. + if bytes.Equal(e1.enc, e2.enc) { + continue + } + // The bytes are different, but the extensions might still be + // equal. We need to decode them to compare. + } + if m1 != nil && m2 != nil { // Both are unencoded. if !equalAny(reflect.ValueOf(m1), reflect.ValueOf(m2), nil) { @@ -276,8 +272,12 @@ func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool { desc = m[extNum] } if desc == nil { + // If both have only encoded form and the bytes are the same, + // it is handled above. We get here when the bytes are different. + // We don't know how to decode it, so just compare them as byte + // slices. log.Printf("proto: don't know how to compare extension %d of %v", extNum, base) - continue + return false } var err error if m1 == nil { diff --git a/vendor/github.com/golang/protobuf/proto/extensions.go b/vendor/github.com/golang/protobuf/proto/extensions.go index eaad2183..816a3b9d 100644 --- a/vendor/github.com/golang/protobuf/proto/extensions.go +++ b/vendor/github.com/golang/protobuf/proto/extensions.go @@ -38,6 +38,7 @@ package proto import ( "errors" "fmt" + "io" "reflect" "strconv" "sync" @@ -91,14 +92,29 @@ func (n notLocker) Unlock() {} // extendable returns the extendableProto interface for the given generated proto message. // If the proto message has the old extension format, it returns a wrapper that implements // the extendableProto interface. -func extendable(p interface{}) (extendableProto, bool) { - if ep, ok := p.(extendableProto); ok { - return ep, ok +func extendable(p interface{}) (extendableProto, error) { + switch p := p.(type) { + case extendableProto: + if isNilPtr(p) { + return nil, fmt.Errorf("proto: nil %T is not extendable", p) + } + return p, nil + case extendableProtoV1: + if isNilPtr(p) { + return nil, fmt.Errorf("proto: nil %T is not extendable", p) + } + return extensionAdapter{p}, nil } - if ep, ok := p.(extendableProtoV1); ok { - return extensionAdapter{ep}, ok - } - return nil, false + // Don't allocate a specific error containing %T: + // this is the hot path for Clone and MarshalText. + return nil, errNotExtendable +} + +var errNotExtendable = errors.New("proto: not an extendable proto.Message") + +func isNilPtr(x interface{}) bool { + v := reflect.ValueOf(x) + return v.Kind() == reflect.Ptr && v.IsNil() } // XXX_InternalExtensions is an internal representation of proto extensions. @@ -143,9 +159,6 @@ func (e *XXX_InternalExtensions) extensionsRead() (map[int32]Extension, sync.Loc return e.p.extensionMap, &e.p.mu } -var extendableProtoType = reflect.TypeOf((*extendableProto)(nil)).Elem() -var extendableProtoV1Type = reflect.TypeOf((*extendableProtoV1)(nil)).Elem() - // ExtensionDesc represents an extension specification. // Used in generated code from the protocol compiler. type ExtensionDesc struct { @@ -179,8 +192,8 @@ type Extension struct { // SetRawExtension is for testing only. func SetRawExtension(base Message, id int32, b []byte) { - epb, ok := extendable(base) - if !ok { + epb, err := extendable(base) + if err != nil { return } extmap := epb.extensionsWrite() @@ -205,7 +218,7 @@ func checkExtensionTypes(pb extendableProto, extension *ExtensionDesc) error { pbi = ea.extendableProtoV1 } if a, b := reflect.TypeOf(pbi), reflect.TypeOf(extension.ExtendedType); a != b { - return errors.New("proto: bad extended type; " + b.String() + " does not extend " + a.String()) + return fmt.Errorf("proto: bad extended type; %v does not extend %v", b, a) } // Check the range. if !isExtensionField(pb, extension.Field) { @@ -250,85 +263,11 @@ func extensionProperties(ed *ExtensionDesc) *Properties { return prop } -// encode encodes any unmarshaled (unencoded) extensions in e. -func encodeExtensions(e *XXX_InternalExtensions) error { - m, mu := e.extensionsRead() - if m == nil { - return nil // fast path - } - mu.Lock() - defer mu.Unlock() - return encodeExtensionsMap(m) -} - -// encode encodes any unmarshaled (unencoded) extensions in e. -func encodeExtensionsMap(m map[int32]Extension) error { - for k, e := range m { - if e.value == nil || e.desc == nil { - // Extension is only in its encoded form. - continue - } - - // We don't skip extensions that have an encoded form set, - // because the extension value may have been mutated after - // the last time this function was called. - - et := reflect.TypeOf(e.desc.ExtensionType) - props := extensionProperties(e.desc) - - p := NewBuffer(nil) - // If e.value has type T, the encoder expects a *struct{ X T }. - // Pass a *T with a zero field and hope it all works out. - x := reflect.New(et) - x.Elem().Set(reflect.ValueOf(e.value)) - if err := props.enc(p, props, toStructPointer(x)); err != nil { - return err - } - e.enc = p.buf - m[k] = e - } - return nil -} - -func extensionsSize(e *XXX_InternalExtensions) (n int) { - m, mu := e.extensionsRead() - if m == nil { - return 0 - } - mu.Lock() - defer mu.Unlock() - return extensionsMapSize(m) -} - -func extensionsMapSize(m map[int32]Extension) (n int) { - for _, e := range m { - if e.value == nil || e.desc == nil { - // Extension is only in its encoded form. - n += len(e.enc) - continue - } - - // We don't skip extensions that have an encoded form set, - // because the extension value may have been mutated after - // the last time this function was called. - - et := reflect.TypeOf(e.desc.ExtensionType) - props := extensionProperties(e.desc) - - // If e.value has type T, the encoder expects a *struct{ X T }. - // Pass a *T with a zero field and hope it all works out. - x := reflect.New(et) - x.Elem().Set(reflect.ValueOf(e.value)) - n += props.size(props, toStructPointer(x)) - } - return -} - // HasExtension returns whether the given extension is present in pb. func HasExtension(pb Message, extension *ExtensionDesc) bool { // TODO: Check types, field numbers, etc.? - epb, ok := extendable(pb) - if !ok { + epb, err := extendable(pb) + if err != nil { return false } extmap, mu := epb.extensionsRead() @@ -336,15 +275,15 @@ func HasExtension(pb Message, extension *ExtensionDesc) bool { return false } mu.Lock() - _, ok = extmap[extension.Field] + _, ok := extmap[extension.Field] mu.Unlock() return ok } // ClearExtension removes the given extension from pb. func ClearExtension(pb Message, extension *ExtensionDesc) { - epb, ok := extendable(pb) - if !ok { + epb, err := extendable(pb) + if err != nil { return } // TODO: Check types, field numbers, etc.? @@ -352,16 +291,26 @@ func ClearExtension(pb Message, extension *ExtensionDesc) { delete(extmap, extension.Field) } -// GetExtension parses and returns the given extension of pb. -// If the extension is not present and has no default value it returns ErrMissingExtension. +// GetExtension retrieves a proto2 extended field from pb. +// +// If the descriptor is type complete (i.e., ExtensionDesc.ExtensionType is non-nil), +// then GetExtension parses the encoded field and returns a Go value of the specified type. +// If the field is not present, then the default value is returned (if one is specified), +// otherwise ErrMissingExtension is reported. +// +// If the descriptor is not type complete (i.e., ExtensionDesc.ExtensionType is nil), +// then GetExtension returns the raw encoded bytes of the field extension. func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) { - epb, ok := extendable(pb) - if !ok { - return nil, errors.New("proto: not an extendable proto") + epb, err := extendable(pb) + if err != nil { + return nil, err } - if err := checkExtensionTypes(epb, extension); err != nil { - return nil, err + if extension.ExtendedType != nil { + // can only check type if this is a complete descriptor + if err := checkExtensionTypes(epb, extension); err != nil { + return nil, err + } } emap, mu := epb.extensionsRead() @@ -388,6 +337,11 @@ func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) { return e.value, nil } + if extension.ExtensionType == nil { + // incomplete descriptor + return e.enc, nil + } + v, err := decodeExtension(e.enc, extension) if err != nil { return nil, err @@ -405,6 +359,11 @@ func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) { // defaultExtensionValue returns the default value for extension. // If no default for an extension is defined ErrMissingExtension is returned. func defaultExtensionValue(extension *ExtensionDesc) (interface{}, error) { + if extension.ExtensionType == nil { + // incomplete descriptor, so no default + return nil, ErrMissingExtension + } + t := reflect.TypeOf(extension.ExtensionType) props := extensionProperties(extension) @@ -439,31 +398,28 @@ func defaultExtensionValue(extension *ExtensionDesc) (interface{}, error) { // decodeExtension decodes an extension encoded in b. func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) { - o := NewBuffer(b) - t := reflect.TypeOf(extension.ExtensionType) - - props := extensionProperties(extension) + unmarshal := typeUnmarshaler(t, extension.Tag) // t is a pointer to a struct, pointer to basic type or a slice. - // Allocate a "field" to store the pointer/slice itself; the - // pointer/slice will be stored here. We pass - // the address of this field to props.dec. - // This passes a zero field and a *t and lets props.dec - // interpret it as a *struct{ x t }. + // Allocate space to store the pointer/slice. value := reflect.New(t).Elem() + var err error for { - // Discard wire type and field number varint. It isn't needed. - if _, err := o.DecodeVarint(); err != nil { + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + wire := int(x) & 7 + + b, err = unmarshal(b, valToPointer(value.Addr()), wire) + if err != nil { return nil, err } - if err := props.dec(o, props, toStructPointer(value.Addr())); err != nil { - return nil, err - } - - if o.index >= len(o.buf) { + if len(b) == 0 { break } } @@ -473,9 +429,9 @@ func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) { // GetExtensions returns a slice of the extensions present in pb that are also listed in es. // The returned slice has the same length as es; missing extensions will appear as nil elements. func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, err error) { - epb, ok := extendable(pb) - if !ok { - return nil, errors.New("proto: not an extendable proto") + epb, err := extendable(pb) + if err != nil { + return nil, err } extensions = make([]interface{}, len(es)) for i, e := range es { @@ -494,9 +450,9 @@ func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, e // For non-registered extensions, ExtensionDescs returns an incomplete descriptor containing // just the Field field, which defines the extension's field number. func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) { - epb, ok := extendable(pb) - if !ok { - return nil, fmt.Errorf("proto: %T is not an extendable proto.Message", pb) + epb, err := extendable(pb) + if err != nil { + return nil, err } registeredExtensions := RegisteredExtensions(pb) @@ -523,9 +479,9 @@ func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) { // SetExtension sets the specified extension of pb to the specified value. func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error { - epb, ok := extendable(pb) - if !ok { - return errors.New("proto: not an extendable proto") + epb, err := extendable(pb) + if err != nil { + return err } if err := checkExtensionTypes(epb, extension); err != nil { return err @@ -550,8 +506,8 @@ func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error // ClearAllExtensions clears all extensions from pb. func ClearAllExtensions(pb Message) { - epb, ok := extendable(pb) - if !ok { + epb, err := extendable(pb) + if err != nil { return } m := epb.extensionsWrite() diff --git a/vendor/github.com/golang/protobuf/proto/lib.go b/vendor/github.com/golang/protobuf/proto/lib.go index ac4ddbc0..0e2191b8 100644 --- a/vendor/github.com/golang/protobuf/proto/lib.go +++ b/vendor/github.com/golang/protobuf/proto/lib.go @@ -73,7 +73,6 @@ for a protocol buffer variable v: When the .proto file specifies `syntax="proto3"`, there are some differences: - Non-repeated fields of non-message type are values instead of pointers. - - Getters are only generated for message and oneof fields. - Enum types do not get an Enum method. The simplest way to describe this is to see an example. @@ -266,6 +265,7 @@ package proto import ( "encoding/json" + "errors" "fmt" "log" "reflect" @@ -274,6 +274,8 @@ import ( "sync" ) +var errInvalidUTF8 = errors.New("proto: invalid UTF-8 string") + // Message is implemented by generated protocol buffer messages. type Message interface { Reset() @@ -310,16 +312,7 @@ type Buffer struct { buf []byte // encode/decode byte stream index int // read point - // pools of basic types to amortize allocation. - bools []bool - uint32s []uint32 - uint64s []uint64 - - // extra pools, only used with pointer_reflect.go - int32s []int32 - int64s []int64 - float32s []float32 - float64s []float64 + deterministic bool } // NewBuffer allocates a new Buffer and initializes its internal data to @@ -344,6 +337,30 @@ func (p *Buffer) SetBuf(s []byte) { // Bytes returns the contents of the Buffer. func (p *Buffer) Bytes() []byte { return p.buf } +// SetDeterministic sets whether to use deterministic serialization. +// +// Deterministic serialization guarantees that for a given binary, equal +// messages will always be serialized to the same bytes. This implies: +// +// - Repeated serialization of a message will return the same bytes. +// - Different processes of the same binary (which may be executing on +// different machines) will serialize equal messages to the same bytes. +// +// Note that the deterministic serialization is NOT canonical across +// languages. It is not guaranteed to remain stable over time. It is unstable +// across different builds with schema changes due to unknown fields. +// Users who need canonical serialization (e.g., persistent storage in a +// canonical form, fingerprinting, etc.) should define their own +// canonicalization specification and implement their own serializer rather +// than relying on this API. +// +// If deterministic serialization is requested, map entries will be sorted +// by keys in lexographical order. This is an implementation detail and +// subject to change. +func (p *Buffer) SetDeterministic(deterministic bool) { + p.deterministic = deterministic +} + /* * Helper routines for simplifying the creation of optional fields of basic type. */ @@ -832,22 +849,12 @@ func fieldDefault(ft reflect.Type, prop *Properties) (sf *scalarField, nestedMes return sf, false, nil } +// mapKeys returns a sort.Interface to be used for sorting the map keys. // Map fields may have key types of non-float scalars, strings and enums. -// The easiest way to sort them in some deterministic order is to use fmt. -// If this turns out to be inefficient we can always consider other options, -// such as doing a Schwartzian transform. - func mapKeys(vs []reflect.Value) sort.Interface { - s := mapKeySorter{ - vs: vs, - // default Less function: textual comparison - less: func(a, b reflect.Value) bool { - return fmt.Sprint(a.Interface()) < fmt.Sprint(b.Interface()) - }, - } + s := mapKeySorter{vs: vs} - // Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps; - // numeric keys are sorted numerically. + // Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps. if len(vs) == 0 { return s } @@ -856,6 +863,12 @@ func mapKeys(vs []reflect.Value) sort.Interface { s.less = func(a, b reflect.Value) bool { return a.Int() < b.Int() } case reflect.Uint32, reflect.Uint64: s.less = func(a, b reflect.Value) bool { return a.Uint() < b.Uint() } + case reflect.Bool: + s.less = func(a, b reflect.Value) bool { return !a.Bool() && b.Bool() } // false < true + case reflect.String: + s.less = func(a, b reflect.Value) bool { return a.String() < b.String() } + default: + panic(fmt.Sprintf("unsupported map key type: %v", vs[0].Kind())) } return s @@ -896,3 +909,13 @@ const ProtoPackageIsVersion2 = true // ProtoPackageIsVersion1 is referenced from generated protocol buffer files // to assert that that code is compatible with this version of the proto package. const ProtoPackageIsVersion1 = true + +// InternalMessageInfo is a type used internally by generated .pb.go files. +// This type is not intended to be used by non-generated code. +// This type is not subject to any compatibility guarantee. +type InternalMessageInfo struct { + marshal *marshalInfo + unmarshal *unmarshalInfo + merge *mergeInfo + discard *discardInfo +} diff --git a/vendor/github.com/golang/protobuf/proto/message_set.go b/vendor/github.com/golang/protobuf/proto/message_set.go index fd982dec..3b6ca41d 100644 --- a/vendor/github.com/golang/protobuf/proto/message_set.go +++ b/vendor/github.com/golang/protobuf/proto/message_set.go @@ -42,6 +42,7 @@ import ( "fmt" "reflect" "sort" + "sync" ) // errNoMessageTypeID occurs when a protocol buffer does not have a message type ID. @@ -94,10 +95,7 @@ func (ms *messageSet) find(pb Message) *_MessageSet_Item { } func (ms *messageSet) Has(pb Message) bool { - if ms.find(pb) != nil { - return true - } - return false + return ms.find(pb) != nil } func (ms *messageSet) Unmarshal(pb Message) error { @@ -150,46 +148,42 @@ func skipVarint(buf []byte) []byte { // MarshalMessageSet encodes the extension map represented by m in the message set wire format. // It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option. func MarshalMessageSet(exts interface{}) ([]byte, error) { - var m map[int32]Extension + return marshalMessageSet(exts, false) +} + +// marshaMessageSet implements above function, with the opt to turn on / off deterministic during Marshal. +func marshalMessageSet(exts interface{}, deterministic bool) ([]byte, error) { switch exts := exts.(type) { case *XXX_InternalExtensions: - if err := encodeExtensions(exts); err != nil { - return nil, err - } - m, _ = exts.extensionsRead() + var u marshalInfo + siz := u.sizeMessageSet(exts) + b := make([]byte, 0, siz) + return u.appendMessageSet(b, exts, deterministic) + case map[int32]Extension: - if err := encodeExtensionsMap(exts); err != nil { - return nil, err + // This is an old-style extension map. + // Wrap it in a new-style XXX_InternalExtensions. + ie := XXX_InternalExtensions{ + p: &struct { + mu sync.Mutex + extensionMap map[int32]Extension + }{ + extensionMap: exts, + }, } - m = exts + + var u marshalInfo + siz := u.sizeMessageSet(&ie) + b := make([]byte, 0, siz) + return u.appendMessageSet(b, &ie, deterministic) + default: return nil, errors.New("proto: not an extension map") } - - // Sort extension IDs to provide a deterministic encoding. - // See also enc_map in encode.go. - ids := make([]int, 0, len(m)) - for id := range m { - ids = append(ids, int(id)) - } - sort.Ints(ids) - - ms := &messageSet{Item: make([]*_MessageSet_Item, 0, len(m))} - for _, id := range ids { - e := m[int32(id)] - // Remove the wire type and field number varint, as well as the length varint. - msg := skipVarint(skipVarint(e.enc)) - - ms.Item = append(ms.Item, &_MessageSet_Item{ - TypeId: Int32(int32(id)), - Message: msg, - }) - } - return Marshal(ms) } // UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format. -// It is called by generated Unmarshal methods on protocol buffer messages with the message_set_wire_format option. +// It is called by Unmarshal methods on protocol buffer messages with the message_set_wire_format option. func UnmarshalMessageSet(buf []byte, exts interface{}) error { var m map[int32]Extension switch exts := exts.(type) { @@ -235,7 +229,15 @@ func MarshalMessageSetJSON(exts interface{}) ([]byte, error) { var m map[int32]Extension switch exts := exts.(type) { case *XXX_InternalExtensions: - m, _ = exts.extensionsRead() + var mu sync.Locker + m, mu = exts.extensionsRead() + if m != nil { + // Keep the extensions map locked until we're done marshaling to prevent + // races between marshaling and unmarshaling the lazily-{en,de}coded + // values. + mu.Lock() + defer mu.Unlock() + } case map[int32]Extension: m = exts default: @@ -253,15 +255,16 @@ func MarshalMessageSetJSON(exts interface{}) ([]byte, error) { for i, id := range ids { ext := m[id] - if i > 0 { - b.WriteByte(',') - } - msd, ok := messageSetMap[id] if !ok { // Unknown type; we can't render it, so skip it. continue } + + if i > 0 && b.Len() > 1 { + b.WriteByte(',') + } + fmt.Fprintf(&b, `"[%s]":`, msd.name) x := ext.value diff --git a/vendor/github.com/golang/protobuf/proto/pointer_reflect.go b/vendor/github.com/golang/protobuf/proto/pointer_reflect.go index fb512e2e..b6cad908 100644 --- a/vendor/github.com/golang/protobuf/proto/pointer_reflect.go +++ b/vendor/github.com/golang/protobuf/proto/pointer_reflect.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build appengine js +// +build purego appengine js // This file contains an implementation of proto field accesses using package reflect. // It is slower than the code in pointer_unsafe.go but it avoids package unsafe and can @@ -38,32 +38,13 @@ package proto import ( - "math" "reflect" + "sync" ) -// A structPointer is a pointer to a struct. -type structPointer struct { - v reflect.Value -} +const unsafeAllowed = false -// toStructPointer returns a structPointer equivalent to the given reflect value. -// The reflect value must itself be a pointer to a struct. -func toStructPointer(v reflect.Value) structPointer { - return structPointer{v} -} - -// IsNil reports whether p is nil. -func structPointer_IsNil(p structPointer) bool { - return p.v.IsNil() -} - -// Interface returns the struct pointer as an interface value. -func structPointer_Interface(p structPointer, _ reflect.Type) interface{} { - return p.v.Interface() -} - -// A field identifies a field in a struct, accessible from a structPointer. +// A field identifies a field in a struct, accessible from a pointer. // In this implementation, a field is identified by the sequence of field indices // passed to reflect's FieldByIndex. type field []int @@ -76,409 +57,301 @@ func toField(f *reflect.StructField) field { // invalidField is an invalid field identifier. var invalidField = field(nil) +// zeroField is a noop when calling pointer.offset. +var zeroField = field([]int{}) + // IsValid reports whether the field identifier is valid. func (f field) IsValid() bool { return f != nil } -// field returns the given field in the struct as a reflect value. -func structPointer_field(p structPointer, f field) reflect.Value { - // Special case: an extension map entry with a value of type T - // passes a *T to the struct-handling code with a zero field, - // expecting that it will be treated as equivalent to *struct{ X T }, - // which has the same memory layout. We have to handle that case - // specially, because reflect will panic if we call FieldByIndex on a - // non-struct. - if f == nil { - return p.v.Elem() - } - - return p.v.Elem().FieldByIndex(f) -} - -// ifield returns the given field in the struct as an interface value. -func structPointer_ifield(p structPointer, f field) interface{} { - return structPointer_field(p, f).Addr().Interface() -} - -// Bytes returns the address of a []byte field in the struct. -func structPointer_Bytes(p structPointer, f field) *[]byte { - return structPointer_ifield(p, f).(*[]byte) -} - -// BytesSlice returns the address of a [][]byte field in the struct. -func structPointer_BytesSlice(p structPointer, f field) *[][]byte { - return structPointer_ifield(p, f).(*[][]byte) -} - -// Bool returns the address of a *bool field in the struct. -func structPointer_Bool(p structPointer, f field) **bool { - return structPointer_ifield(p, f).(**bool) -} - -// BoolVal returns the address of a bool field in the struct. -func structPointer_BoolVal(p structPointer, f field) *bool { - return structPointer_ifield(p, f).(*bool) -} - -// BoolSlice returns the address of a []bool field in the struct. -func structPointer_BoolSlice(p structPointer, f field) *[]bool { - return structPointer_ifield(p, f).(*[]bool) -} - -// String returns the address of a *string field in the struct. -func structPointer_String(p structPointer, f field) **string { - return structPointer_ifield(p, f).(**string) -} - -// StringVal returns the address of a string field in the struct. -func structPointer_StringVal(p structPointer, f field) *string { - return structPointer_ifield(p, f).(*string) -} - -// StringSlice returns the address of a []string field in the struct. -func structPointer_StringSlice(p structPointer, f field) *[]string { - return structPointer_ifield(p, f).(*[]string) -} - -// Extensions returns the address of an extension map field in the struct. -func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions { - return structPointer_ifield(p, f).(*XXX_InternalExtensions) -} - -// ExtMap returns the address of an extension map field in the struct. -func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { - return structPointer_ifield(p, f).(*map[int32]Extension) -} - -// NewAt returns the reflect.Value for a pointer to a field in the struct. -func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value { - return structPointer_field(p, f).Addr() -} - -// SetStructPointer writes a *struct field in the struct. -func structPointer_SetStructPointer(p structPointer, f field, q structPointer) { - structPointer_field(p, f).Set(q.v) -} - -// GetStructPointer reads a *struct field in the struct. -func structPointer_GetStructPointer(p structPointer, f field) structPointer { - return structPointer{structPointer_field(p, f)} -} - -// StructPointerSlice the address of a []*struct field in the struct. -func structPointer_StructPointerSlice(p structPointer, f field) structPointerSlice { - return structPointerSlice{structPointer_field(p, f)} -} - -// A structPointerSlice represents the address of a slice of pointers to structs -// (themselves messages or groups). That is, v.Type() is *[]*struct{...}. -type structPointerSlice struct { +// The pointer type is for the table-driven decoder. +// The implementation here uses a reflect.Value of pointer type to +// create a generic pointer. In pointer_unsafe.go we use unsafe +// instead of reflect to implement the same (but faster) interface. +type pointer struct { v reflect.Value } -func (p structPointerSlice) Len() int { return p.v.Len() } -func (p structPointerSlice) Index(i int) structPointer { return structPointer{p.v.Index(i)} } -func (p structPointerSlice) Append(q structPointer) { - p.v.Set(reflect.Append(p.v, q.v)) +// toPointer converts an interface of pointer type to a pointer +// that points to the same target. +func toPointer(i *Message) pointer { + return pointer{v: reflect.ValueOf(*i)} } -var ( - int32Type = reflect.TypeOf(int32(0)) - uint32Type = reflect.TypeOf(uint32(0)) - float32Type = reflect.TypeOf(float32(0)) - int64Type = reflect.TypeOf(int64(0)) - uint64Type = reflect.TypeOf(uint64(0)) - float64Type = reflect.TypeOf(float64(0)) -) - -// A word32 represents a field of type *int32, *uint32, *float32, or *enum. -// That is, v.Type() is *int32, *uint32, *float32, or *enum and v is assignable. -type word32 struct { - v reflect.Value +// toAddrPointer converts an interface to a pointer that points to +// the interface data. +func toAddrPointer(i *interface{}, isptr bool) pointer { + v := reflect.ValueOf(*i) + u := reflect.New(v.Type()) + u.Elem().Set(v) + return pointer{v: u} } -// IsNil reports whether p is nil. -func word32_IsNil(p word32) bool { +// valToPointer converts v to a pointer. v must be of pointer type. +func valToPointer(v reflect.Value) pointer { + return pointer{v: v} +} + +// offset converts from a pointer to a structure to a pointer to +// one of its fields. +func (p pointer) offset(f field) pointer { + return pointer{v: p.v.Elem().FieldByIndex(f).Addr()} +} + +func (p pointer) isNil() bool { return p.v.IsNil() } -// Set sets p to point at a newly allocated word with bits set to x. -func word32_Set(p word32, o *Buffer, x uint32) { - t := p.v.Type().Elem() - switch t { - case int32Type: - if len(o.int32s) == 0 { - o.int32s = make([]int32, uint32PoolSize) - } - o.int32s[0] = int32(x) - p.v.Set(reflect.ValueOf(&o.int32s[0])) - o.int32s = o.int32s[1:] - return - case uint32Type: - if len(o.uint32s) == 0 { - o.uint32s = make([]uint32, uint32PoolSize) - } - o.uint32s[0] = x - p.v.Set(reflect.ValueOf(&o.uint32s[0])) - o.uint32s = o.uint32s[1:] - return - case float32Type: - if len(o.float32s) == 0 { - o.float32s = make([]float32, uint32PoolSize) - } - o.float32s[0] = math.Float32frombits(x) - p.v.Set(reflect.ValueOf(&o.float32s[0])) - o.float32s = o.float32s[1:] - return - } - - // must be enum - p.v.Set(reflect.New(t)) - p.v.Elem().SetInt(int64(int32(x))) -} - -// Get gets the bits pointed at by p, as a uint32. -func word32_Get(p word32) uint32 { - elem := p.v.Elem() - switch elem.Kind() { - case reflect.Int32: - return uint32(elem.Int()) - case reflect.Uint32: - return uint32(elem.Uint()) - case reflect.Float32: - return math.Float32bits(float32(elem.Float())) - } - panic("unreachable") -} - -// Word32 returns a reference to a *int32, *uint32, *float32, or *enum field in the struct. -func structPointer_Word32(p structPointer, f field) word32 { - return word32{structPointer_field(p, f)} -} - -// A word32Val represents a field of type int32, uint32, float32, or enum. -// That is, v.Type() is int32, uint32, float32, or enum and v is assignable. -type word32Val struct { - v reflect.Value -} - -// Set sets *p to x. -func word32Val_Set(p word32Val, x uint32) { - switch p.v.Type() { - case int32Type: - p.v.SetInt(int64(x)) - return - case uint32Type: - p.v.SetUint(uint64(x)) - return - case float32Type: - p.v.SetFloat(float64(math.Float32frombits(x))) - return - } - - // must be enum - p.v.SetInt(int64(int32(x))) -} - -// Get gets the bits pointed at by p, as a uint32. -func word32Val_Get(p word32Val) uint32 { - elem := p.v - switch elem.Kind() { - case reflect.Int32: - return uint32(elem.Int()) - case reflect.Uint32: - return uint32(elem.Uint()) - case reflect.Float32: - return math.Float32bits(float32(elem.Float())) - } - panic("unreachable") -} - -// Word32Val returns a reference to a int32, uint32, float32, or enum field in the struct. -func structPointer_Word32Val(p structPointer, f field) word32Val { - return word32Val{structPointer_field(p, f)} -} - -// A word32Slice is a slice of 32-bit values. -// That is, v.Type() is []int32, []uint32, []float32, or []enum. -type word32Slice struct { - v reflect.Value -} - -func (p word32Slice) Append(x uint32) { - n, m := p.v.Len(), p.v.Cap() +// grow updates the slice s in place to make it one element longer. +// s must be addressable. +// Returns the (addressable) new element. +func grow(s reflect.Value) reflect.Value { + n, m := s.Len(), s.Cap() if n < m { - p.v.SetLen(n + 1) + s.SetLen(n + 1) } else { - t := p.v.Type().Elem() - p.v.Set(reflect.Append(p.v, reflect.Zero(t))) + s.Set(reflect.Append(s, reflect.Zero(s.Type().Elem()))) } - elem := p.v.Index(n) - switch elem.Kind() { - case reflect.Int32: - elem.SetInt(int64(int32(x))) - case reflect.Uint32: - elem.SetUint(uint64(x)) - case reflect.Float32: - elem.SetFloat(float64(math.Float32frombits(x))) + return s.Index(n) +} + +func (p pointer) toInt64() *int64 { + return p.v.Interface().(*int64) +} +func (p pointer) toInt64Ptr() **int64 { + return p.v.Interface().(**int64) +} +func (p pointer) toInt64Slice() *[]int64 { + return p.v.Interface().(*[]int64) +} + +var int32ptr = reflect.TypeOf((*int32)(nil)) + +func (p pointer) toInt32() *int32 { + return p.v.Convert(int32ptr).Interface().(*int32) +} + +// The toInt32Ptr/Slice methods don't work because of enums. +// Instead, we must use set/get methods for the int32ptr/slice case. +/* + func (p pointer) toInt32Ptr() **int32 { + return p.v.Interface().(**int32) +} + func (p pointer) toInt32Slice() *[]int32 { + return p.v.Interface().(*[]int32) +} +*/ +func (p pointer) getInt32Ptr() *int32 { + if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) { + // raw int32 type + return p.v.Elem().Interface().(*int32) } + // an enum + return p.v.Elem().Convert(int32PtrType).Interface().(*int32) +} +func (p pointer) setInt32Ptr(v int32) { + // Allocate value in a *int32. Possibly convert that to a *enum. + // Then assign it to a **int32 or **enum. + // Note: we can convert *int32 to *enum, but we can't convert + // **int32 to **enum! + p.v.Elem().Set(reflect.ValueOf(&v).Convert(p.v.Type().Elem())) } -func (p word32Slice) Len() int { - return p.v.Len() -} - -func (p word32Slice) Index(i int) uint32 { - elem := p.v.Index(i) - switch elem.Kind() { - case reflect.Int32: - return uint32(elem.Int()) - case reflect.Uint32: - return uint32(elem.Uint()) - case reflect.Float32: - return math.Float32bits(float32(elem.Float())) +// getInt32Slice copies []int32 from p as a new slice. +// This behavior differs from the implementation in pointer_unsafe.go. +func (p pointer) getInt32Slice() []int32 { + if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) { + // raw int32 type + return p.v.Elem().Interface().([]int32) } - panic("unreachable") + // an enum + // Allocate a []int32, then assign []enum's values into it. + // Note: we can't convert []enum to []int32. + slice := p.v.Elem() + s := make([]int32, slice.Len()) + for i := 0; i < slice.Len(); i++ { + s[i] = int32(slice.Index(i).Int()) + } + return s } -// Word32Slice returns a reference to a []int32, []uint32, []float32, or []enum field in the struct. -func structPointer_Word32Slice(p structPointer, f field) word32Slice { - return word32Slice{structPointer_field(p, f)} -} - -// word64 is like word32 but for 64-bit values. -type word64 struct { - v reflect.Value -} - -func word64_Set(p word64, o *Buffer, x uint64) { - t := p.v.Type().Elem() - switch t { - case int64Type: - if len(o.int64s) == 0 { - o.int64s = make([]int64, uint64PoolSize) - } - o.int64s[0] = int64(x) - p.v.Set(reflect.ValueOf(&o.int64s[0])) - o.int64s = o.int64s[1:] - return - case uint64Type: - if len(o.uint64s) == 0 { - o.uint64s = make([]uint64, uint64PoolSize) - } - o.uint64s[0] = x - p.v.Set(reflect.ValueOf(&o.uint64s[0])) - o.uint64s = o.uint64s[1:] - return - case float64Type: - if len(o.float64s) == 0 { - o.float64s = make([]float64, uint64PoolSize) - } - o.float64s[0] = math.Float64frombits(x) - p.v.Set(reflect.ValueOf(&o.float64s[0])) - o.float64s = o.float64s[1:] +// setInt32Slice copies []int32 into p as a new slice. +// This behavior differs from the implementation in pointer_unsafe.go. +func (p pointer) setInt32Slice(v []int32) { + if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) { + // raw int32 type + p.v.Elem().Set(reflect.ValueOf(v)) return } - panic("unreachable") -} - -func word64_IsNil(p word64) bool { - return p.v.IsNil() -} - -func word64_Get(p word64) uint64 { - elem := p.v.Elem() - switch elem.Kind() { - case reflect.Int64: - return uint64(elem.Int()) - case reflect.Uint64: - return elem.Uint() - case reflect.Float64: - return math.Float64bits(elem.Float()) + // an enum + // Allocate a []enum, then assign []int32's values into it. + // Note: we can't convert []enum to []int32. + slice := reflect.MakeSlice(p.v.Type().Elem(), len(v), cap(v)) + for i, x := range v { + slice.Index(i).SetInt(int64(x)) } - panic("unreachable") + p.v.Elem().Set(slice) +} +func (p pointer) appendInt32Slice(v int32) { + grow(p.v.Elem()).SetInt(int64(v)) } -func structPointer_Word64(p structPointer, f field) word64 { - return word64{structPointer_field(p, f)} +func (p pointer) toUint64() *uint64 { + return p.v.Interface().(*uint64) +} +func (p pointer) toUint64Ptr() **uint64 { + return p.v.Interface().(**uint64) +} +func (p pointer) toUint64Slice() *[]uint64 { + return p.v.Interface().(*[]uint64) +} +func (p pointer) toUint32() *uint32 { + return p.v.Interface().(*uint32) +} +func (p pointer) toUint32Ptr() **uint32 { + return p.v.Interface().(**uint32) +} +func (p pointer) toUint32Slice() *[]uint32 { + return p.v.Interface().(*[]uint32) +} +func (p pointer) toBool() *bool { + return p.v.Interface().(*bool) +} +func (p pointer) toBoolPtr() **bool { + return p.v.Interface().(**bool) +} +func (p pointer) toBoolSlice() *[]bool { + return p.v.Interface().(*[]bool) +} +func (p pointer) toFloat64() *float64 { + return p.v.Interface().(*float64) +} +func (p pointer) toFloat64Ptr() **float64 { + return p.v.Interface().(**float64) +} +func (p pointer) toFloat64Slice() *[]float64 { + return p.v.Interface().(*[]float64) +} +func (p pointer) toFloat32() *float32 { + return p.v.Interface().(*float32) +} +func (p pointer) toFloat32Ptr() **float32 { + return p.v.Interface().(**float32) +} +func (p pointer) toFloat32Slice() *[]float32 { + return p.v.Interface().(*[]float32) +} +func (p pointer) toString() *string { + return p.v.Interface().(*string) +} +func (p pointer) toStringPtr() **string { + return p.v.Interface().(**string) +} +func (p pointer) toStringSlice() *[]string { + return p.v.Interface().(*[]string) +} +func (p pointer) toBytes() *[]byte { + return p.v.Interface().(*[]byte) +} +func (p pointer) toBytesSlice() *[][]byte { + return p.v.Interface().(*[][]byte) +} +func (p pointer) toExtensions() *XXX_InternalExtensions { + return p.v.Interface().(*XXX_InternalExtensions) +} +func (p pointer) toOldExtensions() *map[int32]Extension { + return p.v.Interface().(*map[int32]Extension) +} +func (p pointer) getPointer() pointer { + return pointer{v: p.v.Elem()} +} +func (p pointer) setPointer(q pointer) { + p.v.Elem().Set(q.v) +} +func (p pointer) appendPointer(q pointer) { + grow(p.v.Elem()).Set(q.v) } -// word64Val is like word32Val but for 64-bit values. -type word64Val struct { - v reflect.Value +// getPointerSlice copies []*T from p as a new []pointer. +// This behavior differs from the implementation in pointer_unsafe.go. +func (p pointer) getPointerSlice() []pointer { + if p.v.IsNil() { + return nil + } + n := p.v.Elem().Len() + s := make([]pointer, n) + for i := 0; i < n; i++ { + s[i] = pointer{v: p.v.Elem().Index(i)} + } + return s } -func word64Val_Set(p word64Val, o *Buffer, x uint64) { - switch p.v.Type() { - case int64Type: - p.v.SetInt(int64(x)) - return - case uint64Type: - p.v.SetUint(x) - return - case float64Type: - p.v.SetFloat(math.Float64frombits(x)) +// setPointerSlice copies []pointer into p as a new []*T. +// This behavior differs from the implementation in pointer_unsafe.go. +func (p pointer) setPointerSlice(v []pointer) { + if v == nil { + p.v.Elem().Set(reflect.New(p.v.Elem().Type()).Elem()) return } - panic("unreachable") -} - -func word64Val_Get(p word64Val) uint64 { - elem := p.v - switch elem.Kind() { - case reflect.Int64: - return uint64(elem.Int()) - case reflect.Uint64: - return elem.Uint() - case reflect.Float64: - return math.Float64bits(elem.Float()) + s := reflect.MakeSlice(p.v.Elem().Type(), 0, len(v)) + for _, p := range v { + s = reflect.Append(s, p.v) } - panic("unreachable") + p.v.Elem().Set(s) } -func structPointer_Word64Val(p structPointer, f field) word64Val { - return word64Val{structPointer_field(p, f)} -} - -type word64Slice struct { - v reflect.Value -} - -func (p word64Slice) Append(x uint64) { - n, m := p.v.Len(), p.v.Cap() - if n < m { - p.v.SetLen(n + 1) - } else { - t := p.v.Type().Elem() - p.v.Set(reflect.Append(p.v, reflect.Zero(t))) - } - elem := p.v.Index(n) - switch elem.Kind() { - case reflect.Int64: - elem.SetInt(int64(int64(x))) - case reflect.Uint64: - elem.SetUint(uint64(x)) - case reflect.Float64: - elem.SetFloat(float64(math.Float64frombits(x))) +// getInterfacePointer returns a pointer that points to the +// interface data of the interface pointed by p. +func (p pointer) getInterfacePointer() pointer { + if p.v.Elem().IsNil() { + return pointer{v: p.v.Elem()} } + return pointer{v: p.v.Elem().Elem().Elem().Field(0).Addr()} // *interface -> interface -> *struct -> struct } -func (p word64Slice) Len() int { - return p.v.Len() +func (p pointer) asPointerTo(t reflect.Type) reflect.Value { + // TODO: check that p.v.Type().Elem() == t? + return p.v } -func (p word64Slice) Index(i int) uint64 { - elem := p.v.Index(i) - switch elem.Kind() { - case reflect.Int64: - return uint64(elem.Int()) - case reflect.Uint64: - return uint64(elem.Uint()) - case reflect.Float64: - return math.Float64bits(float64(elem.Float())) - } - panic("unreachable") +func atomicLoadUnmarshalInfo(p **unmarshalInfo) *unmarshalInfo { + atomicLock.Lock() + defer atomicLock.Unlock() + return *p +} +func atomicStoreUnmarshalInfo(p **unmarshalInfo, v *unmarshalInfo) { + atomicLock.Lock() + defer atomicLock.Unlock() + *p = v +} +func atomicLoadMarshalInfo(p **marshalInfo) *marshalInfo { + atomicLock.Lock() + defer atomicLock.Unlock() + return *p +} +func atomicStoreMarshalInfo(p **marshalInfo, v *marshalInfo) { + atomicLock.Lock() + defer atomicLock.Unlock() + *p = v +} +func atomicLoadMergeInfo(p **mergeInfo) *mergeInfo { + atomicLock.Lock() + defer atomicLock.Unlock() + return *p +} +func atomicStoreMergeInfo(p **mergeInfo, v *mergeInfo) { + atomicLock.Lock() + defer atomicLock.Unlock() + *p = v +} +func atomicLoadDiscardInfo(p **discardInfo) *discardInfo { + atomicLock.Lock() + defer atomicLock.Unlock() + return *p +} +func atomicStoreDiscardInfo(p **discardInfo, v *discardInfo) { + atomicLock.Lock() + defer atomicLock.Unlock() + *p = v } -func structPointer_Word64Slice(p structPointer, f field) word64Slice { - return word64Slice{structPointer_field(p, f)} -} +var atomicLock sync.Mutex diff --git a/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go b/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go index 6b5567d4..d55a335d 100644 --- a/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go +++ b/vendor/github.com/golang/protobuf/proto/pointer_unsafe.go @@ -29,7 +29,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +build !appengine,!js +// +build !purego,!appengine,!js // This file contains the implementation of the proto field accesses using package unsafe. @@ -37,38 +37,13 @@ package proto import ( "reflect" + "sync/atomic" "unsafe" ) -// NOTE: These type_Foo functions would more idiomatically be methods, -// but Go does not allow methods on pointer types, and we must preserve -// some pointer type for the garbage collector. We use these -// funcs with clunky names as our poor approximation to methods. -// -// An alternative would be -// type structPointer struct { p unsafe.Pointer } -// but that does not registerize as well. +const unsafeAllowed = true -// A structPointer is a pointer to a struct. -type structPointer unsafe.Pointer - -// toStructPointer returns a structPointer equivalent to the given reflect value. -func toStructPointer(v reflect.Value) structPointer { - return structPointer(unsafe.Pointer(v.Pointer())) -} - -// IsNil reports whether p is nil. -func structPointer_IsNil(p structPointer) bool { - return p == nil -} - -// Interface returns the struct pointer, assumed to have element type t, -// as an interface value. -func structPointer_Interface(p structPointer, t reflect.Type) interface{} { - return reflect.NewAt(t, unsafe.Pointer(p)).Interface() -} - -// A field identifies a field in a struct, accessible from a structPointer. +// A field identifies a field in a struct, accessible from a pointer. // In this implementation, a field is identified by its byte offset from the start of the struct. type field uintptr @@ -80,191 +55,254 @@ func toField(f *reflect.StructField) field { // invalidField is an invalid field identifier. const invalidField = ^field(0) +// zeroField is a noop when calling pointer.offset. +const zeroField = field(0) + // IsValid reports whether the field identifier is valid. func (f field) IsValid() bool { - return f != ^field(0) + return f != invalidField } -// Bytes returns the address of a []byte field in the struct. -func structPointer_Bytes(p structPointer, f field) *[]byte { - return (*[]byte)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// The pointer type below is for the new table-driven encoder/decoder. +// The implementation here uses unsafe.Pointer to create a generic pointer. +// In pointer_reflect.go we use reflect instead of unsafe to implement +// the same (but slower) interface. +type pointer struct { + p unsafe.Pointer } -// BytesSlice returns the address of a [][]byte field in the struct. -func structPointer_BytesSlice(p structPointer, f field) *[][]byte { - return (*[][]byte)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// size of pointer +var ptrSize = unsafe.Sizeof(uintptr(0)) + +// toPointer converts an interface of pointer type to a pointer +// that points to the same target. +func toPointer(i *Message) pointer { + // Super-tricky - read pointer out of data word of interface value. + // Saves ~25ns over the equivalent: + // return valToPointer(reflect.ValueOf(*i)) + return pointer{p: (*[2]unsafe.Pointer)(unsafe.Pointer(i))[1]} } -// Bool returns the address of a *bool field in the struct. -func structPointer_Bool(p structPointer, f field) **bool { - return (**bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// BoolVal returns the address of a bool field in the struct. -func structPointer_BoolVal(p structPointer, f field) *bool { - return (*bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// BoolSlice returns the address of a []bool field in the struct. -func structPointer_BoolSlice(p structPointer, f field) *[]bool { - return (*[]bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// String returns the address of a *string field in the struct. -func structPointer_String(p structPointer, f field) **string { - return (**string)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// StringVal returns the address of a string field in the struct. -func structPointer_StringVal(p structPointer, f field) *string { - return (*string)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// StringSlice returns the address of a []string field in the struct. -func structPointer_StringSlice(p structPointer, f field) *[]string { - return (*[]string)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// ExtMap returns the address of an extension map field in the struct. -func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions { - return (*XXX_InternalExtensions)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension { - return (*map[int32]Extension)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// NewAt returns the reflect.Value for a pointer to a field in the struct. -func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value { - return reflect.NewAt(typ, unsafe.Pointer(uintptr(p)+uintptr(f))) -} - -// SetStructPointer writes a *struct field in the struct. -func structPointer_SetStructPointer(p structPointer, f field, q structPointer) { - *(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f))) = q -} - -// GetStructPointer reads a *struct field in the struct. -func structPointer_GetStructPointer(p structPointer, f field) structPointer { - return *(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// StructPointerSlice the address of a []*struct field in the struct. -func structPointer_StructPointerSlice(p structPointer, f field) *structPointerSlice { - return (*structPointerSlice)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// A structPointerSlice represents a slice of pointers to structs (themselves submessages or groups). -type structPointerSlice []structPointer - -func (v *structPointerSlice) Len() int { return len(*v) } -func (v *structPointerSlice) Index(i int) structPointer { return (*v)[i] } -func (v *structPointerSlice) Append(p structPointer) { *v = append(*v, p) } - -// A word32 is the address of a "pointer to 32-bit value" field. -type word32 **uint32 - -// IsNil reports whether *v is nil. -func word32_IsNil(p word32) bool { - return *p == nil -} - -// Set sets *v to point at a newly allocated word set to x. -func word32_Set(p word32, o *Buffer, x uint32) { - if len(o.uint32s) == 0 { - o.uint32s = make([]uint32, uint32PoolSize) +// toAddrPointer converts an interface to a pointer that points to +// the interface data. +func toAddrPointer(i *interface{}, isptr bool) pointer { + // Super-tricky - read or get the address of data word of interface value. + if isptr { + // The interface is of pointer type, thus it is a direct interface. + // The data word is the pointer data itself. We take its address. + return pointer{p: unsafe.Pointer(uintptr(unsafe.Pointer(i)) + ptrSize)} } - o.uint32s[0] = x - *p = &o.uint32s[0] - o.uint32s = o.uint32s[1:] + // The interface is not of pointer type. The data word is the pointer + // to the data. + return pointer{p: (*[2]unsafe.Pointer)(unsafe.Pointer(i))[1]} } -// Get gets the value pointed at by *v. -func word32_Get(p word32) uint32 { - return **p +// valToPointer converts v to a pointer. v must be of pointer type. +func valToPointer(v reflect.Value) pointer { + return pointer{p: unsafe.Pointer(v.Pointer())} } -// Word32 returns the address of a *int32, *uint32, *float32, or *enum field in the struct. -func structPointer_Word32(p structPointer, f field) word32 { - return word32((**uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +// offset converts from a pointer to a structure to a pointer to +// one of its fields. +func (p pointer) offset(f field) pointer { + // For safety, we should panic if !f.IsValid, however calling panic causes + // this to no longer be inlineable, which is a serious performance cost. + /* + if !f.IsValid() { + panic("invalid field") + } + */ + return pointer{p: unsafe.Pointer(uintptr(p.p) + uintptr(f))} } -// A word32Val is the address of a 32-bit value field. -type word32Val *uint32 - -// Set sets *p to x. -func word32Val_Set(p word32Val, x uint32) { - *p = x +func (p pointer) isNil() bool { + return p.p == nil } -// Get gets the value pointed at by p. -func word32Val_Get(p word32Val) uint32 { - return *p +func (p pointer) toInt64() *int64 { + return (*int64)(p.p) +} +func (p pointer) toInt64Ptr() **int64 { + return (**int64)(p.p) +} +func (p pointer) toInt64Slice() *[]int64 { + return (*[]int64)(p.p) +} +func (p pointer) toInt32() *int32 { + return (*int32)(p.p) } -// Word32Val returns the address of a *int32, *uint32, *float32, or *enum field in the struct. -func structPointer_Word32Val(p structPointer, f field) word32Val { - return word32Val((*uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) -} - -// A word32Slice is a slice of 32-bit values. -type word32Slice []uint32 - -func (v *word32Slice) Append(x uint32) { *v = append(*v, x) } -func (v *word32Slice) Len() int { return len(*v) } -func (v *word32Slice) Index(i int) uint32 { return (*v)[i] } - -// Word32Slice returns the address of a []int32, []uint32, []float32, or []enum field in the struct. -func structPointer_Word32Slice(p structPointer, f field) *word32Slice { - return (*word32Slice)(unsafe.Pointer(uintptr(p) + uintptr(f))) -} - -// word64 is like word32 but for 64-bit values. -type word64 **uint64 - -func word64_Set(p word64, o *Buffer, x uint64) { - if len(o.uint64s) == 0 { - o.uint64s = make([]uint64, uint64PoolSize) +// See pointer_reflect.go for why toInt32Ptr/Slice doesn't exist. +/* + func (p pointer) toInt32Ptr() **int32 { + return (**int32)(p.p) } - o.uint64s[0] = x - *p = &o.uint64s[0] - o.uint64s = o.uint64s[1:] + func (p pointer) toInt32Slice() *[]int32 { + return (*[]int32)(p.p) + } +*/ +func (p pointer) getInt32Ptr() *int32 { + return *(**int32)(p.p) +} +func (p pointer) setInt32Ptr(v int32) { + *(**int32)(p.p) = &v } -func word64_IsNil(p word64) bool { - return *p == nil +// getInt32Slice loads a []int32 from p. +// The value returned is aliased with the original slice. +// This behavior differs from the implementation in pointer_reflect.go. +func (p pointer) getInt32Slice() []int32 { + return *(*[]int32)(p.p) } -func word64_Get(p word64) uint64 { - return **p +// setInt32Slice stores a []int32 to p. +// The value set is aliased with the input slice. +// This behavior differs from the implementation in pointer_reflect.go. +func (p pointer) setInt32Slice(v []int32) { + *(*[]int32)(p.p) = v } -func structPointer_Word64(p structPointer, f field) word64 { - return word64((**uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +// TODO: Can we get rid of appendInt32Slice and use setInt32Slice instead? +func (p pointer) appendInt32Slice(v int32) { + s := (*[]int32)(p.p) + *s = append(*s, v) } -// word64Val is like word32Val but for 64-bit values. -type word64Val *uint64 - -func word64Val_Set(p word64Val, o *Buffer, x uint64) { - *p = x +func (p pointer) toUint64() *uint64 { + return (*uint64)(p.p) +} +func (p pointer) toUint64Ptr() **uint64 { + return (**uint64)(p.p) +} +func (p pointer) toUint64Slice() *[]uint64 { + return (*[]uint64)(p.p) +} +func (p pointer) toUint32() *uint32 { + return (*uint32)(p.p) +} +func (p pointer) toUint32Ptr() **uint32 { + return (**uint32)(p.p) +} +func (p pointer) toUint32Slice() *[]uint32 { + return (*[]uint32)(p.p) +} +func (p pointer) toBool() *bool { + return (*bool)(p.p) +} +func (p pointer) toBoolPtr() **bool { + return (**bool)(p.p) +} +func (p pointer) toBoolSlice() *[]bool { + return (*[]bool)(p.p) +} +func (p pointer) toFloat64() *float64 { + return (*float64)(p.p) +} +func (p pointer) toFloat64Ptr() **float64 { + return (**float64)(p.p) +} +func (p pointer) toFloat64Slice() *[]float64 { + return (*[]float64)(p.p) +} +func (p pointer) toFloat32() *float32 { + return (*float32)(p.p) +} +func (p pointer) toFloat32Ptr() **float32 { + return (**float32)(p.p) +} +func (p pointer) toFloat32Slice() *[]float32 { + return (*[]float32)(p.p) +} +func (p pointer) toString() *string { + return (*string)(p.p) +} +func (p pointer) toStringPtr() **string { + return (**string)(p.p) +} +func (p pointer) toStringSlice() *[]string { + return (*[]string)(p.p) +} +func (p pointer) toBytes() *[]byte { + return (*[]byte)(p.p) +} +func (p pointer) toBytesSlice() *[][]byte { + return (*[][]byte)(p.p) +} +func (p pointer) toExtensions() *XXX_InternalExtensions { + return (*XXX_InternalExtensions)(p.p) +} +func (p pointer) toOldExtensions() *map[int32]Extension { + return (*map[int32]Extension)(p.p) } -func word64Val_Get(p word64Val) uint64 { - return *p +// getPointerSlice loads []*T from p as a []pointer. +// The value returned is aliased with the original slice. +// This behavior differs from the implementation in pointer_reflect.go. +func (p pointer) getPointerSlice() []pointer { + // Super-tricky - p should point to a []*T where T is a + // message type. We load it as []pointer. + return *(*[]pointer)(p.p) } -func structPointer_Word64Val(p structPointer, f field) word64Val { - return word64Val((*uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +// setPointerSlice stores []pointer into p as a []*T. +// The value set is aliased with the input slice. +// This behavior differs from the implementation in pointer_reflect.go. +func (p pointer) setPointerSlice(v []pointer) { + // Super-tricky - p should point to a []*T where T is a + // message type. We store it as []pointer. + *(*[]pointer)(p.p) = v } -// word64Slice is like word32Slice but for 64-bit values. -type word64Slice []uint64 - -func (v *word64Slice) Append(x uint64) { *v = append(*v, x) } -func (v *word64Slice) Len() int { return len(*v) } -func (v *word64Slice) Index(i int) uint64 { return (*v)[i] } - -func structPointer_Word64Slice(p structPointer, f field) *word64Slice { - return (*word64Slice)(unsafe.Pointer(uintptr(p) + uintptr(f))) +// getPointer loads the pointer at p and returns it. +func (p pointer) getPointer() pointer { + return pointer{p: *(*unsafe.Pointer)(p.p)} +} + +// setPointer stores the pointer q at p. +func (p pointer) setPointer(q pointer) { + *(*unsafe.Pointer)(p.p) = q.p +} + +// append q to the slice pointed to by p. +func (p pointer) appendPointer(q pointer) { + s := (*[]unsafe.Pointer)(p.p) + *s = append(*s, q.p) +} + +// getInterfacePointer returns a pointer that points to the +// interface data of the interface pointed by p. +func (p pointer) getInterfacePointer() pointer { + // Super-tricky - read pointer out of data word of interface value. + return pointer{p: (*(*[2]unsafe.Pointer)(p.p))[1]} +} + +// asPointerTo returns a reflect.Value that is a pointer to an +// object of type t stored at p. +func (p pointer) asPointerTo(t reflect.Type) reflect.Value { + return reflect.NewAt(t, p.p) +} + +func atomicLoadUnmarshalInfo(p **unmarshalInfo) *unmarshalInfo { + return (*unmarshalInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func atomicStoreUnmarshalInfo(p **unmarshalInfo, v *unmarshalInfo) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v)) +} +func atomicLoadMarshalInfo(p **marshalInfo) *marshalInfo { + return (*marshalInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func atomicStoreMarshalInfo(p **marshalInfo, v *marshalInfo) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v)) +} +func atomicLoadMergeInfo(p **mergeInfo) *mergeInfo { + return (*mergeInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func atomicStoreMergeInfo(p **mergeInfo, v *mergeInfo) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v)) +} +func atomicLoadDiscardInfo(p **discardInfo) *discardInfo { + return (*discardInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p)))) +} +func atomicStoreDiscardInfo(p **discardInfo, v *discardInfo) { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v)) } diff --git a/vendor/github.com/golang/protobuf/proto/properties.go b/vendor/github.com/golang/protobuf/proto/properties.go index ec2289c0..f710adab 100644 --- a/vendor/github.com/golang/protobuf/proto/properties.go +++ b/vendor/github.com/golang/protobuf/proto/properties.go @@ -58,42 +58,6 @@ const ( WireFixed32 = 5 ) -const startSize = 10 // initial slice/string sizes - -// Encoders are defined in encode.go -// An encoder outputs the full representation of a field, including its -// tag and encoder type. -type encoder func(p *Buffer, prop *Properties, base structPointer) error - -// A valueEncoder encodes a single integer in a particular encoding. -type valueEncoder func(o *Buffer, x uint64) error - -// Sizers are defined in encode.go -// A sizer returns the encoded size of a field, including its tag and encoder -// type. -type sizer func(prop *Properties, base structPointer) int - -// A valueSizer returns the encoded size of a single integer in a particular -// encoding. -type valueSizer func(x uint64) int - -// Decoders are defined in decode.go -// A decoder creates a value from its wire representation. -// Unrecognized subelements are saved in unrec. -type decoder func(p *Buffer, prop *Properties, base structPointer) error - -// A valueDecoder decodes a single integer in a particular encoding. -type valueDecoder func(o *Buffer) (x uint64, err error) - -// A oneofMarshaler does the marshaling for all oneof fields in a message. -type oneofMarshaler func(Message, *Buffer) error - -// A oneofUnmarshaler does the unmarshaling for a oneof field in a message. -type oneofUnmarshaler func(Message, int, int, *Buffer) (bool, error) - -// A oneofSizer does the sizing for all oneof fields in a message. -type oneofSizer func(Message) int - // tagMap is an optimization over map[int]int for typical protocol buffer // use-cases. Encoded protocol buffers are often in tag order with small tag // numbers. @@ -140,13 +104,6 @@ type StructProperties struct { decoderTags tagMap // map from proto tag to struct field number decoderOrigNames map[string]int // map from original name to struct field number order []int // list of struct field numbers in tag order - unrecField field // field id of the XXX_unrecognized []byte field - extendable bool // is this an extendable proto - - oneofMarshaler oneofMarshaler - oneofUnmarshaler oneofUnmarshaler - oneofSizer oneofSizer - stype reflect.Type // OneofTypes contains information about the oneof fields in this message. // It is keyed by the original name of a field. @@ -187,36 +144,19 @@ type Properties struct { Default string // default value HasDefault bool // whether an explicit default was provided - def_uint64 uint64 - enc encoder - valEnc valueEncoder // set for bool and numeric types only - field field - tagcode []byte // encoding of EncodeVarint((Tag<<3)|WireType) - tagbuf [8]byte - stype reflect.Type // set for struct types only - sprop *StructProperties // set for struct types only - isMarshaler bool - isUnmarshaler bool + stype reflect.Type // set for struct types only + sprop *StructProperties // set for struct types only mtype reflect.Type // set for map types only mkeyprop *Properties // set for map types only mvalprop *Properties // set for map types only - - size sizer - valSize valueSizer // set for bool and numeric types only - - dec decoder - valDec valueDecoder // set for bool and numeric types only - - // If this is a packable field, this will be the decoder for the packed version of the field. - packedDec decoder } // String formats the properties in the protobuf struct field tag style. func (p *Properties) String() string { s := p.Wire - s = "," + s += "," s += strconv.Itoa(p.Tag) if p.Required { s += ",req" @@ -262,29 +202,14 @@ func (p *Properties) Parse(s string) { switch p.Wire { case "varint": p.WireType = WireVarint - p.valEnc = (*Buffer).EncodeVarint - p.valDec = (*Buffer).DecodeVarint - p.valSize = sizeVarint case "fixed32": p.WireType = WireFixed32 - p.valEnc = (*Buffer).EncodeFixed32 - p.valDec = (*Buffer).DecodeFixed32 - p.valSize = sizeFixed32 case "fixed64": p.WireType = WireFixed64 - p.valEnc = (*Buffer).EncodeFixed64 - p.valDec = (*Buffer).DecodeFixed64 - p.valSize = sizeFixed64 case "zigzag32": p.WireType = WireVarint - p.valEnc = (*Buffer).EncodeZigzag32 - p.valDec = (*Buffer).DecodeZigzag32 - p.valSize = sizeZigzag32 case "zigzag64": p.WireType = WireVarint - p.valEnc = (*Buffer).EncodeZigzag64 - p.valDec = (*Buffer).DecodeZigzag64 - p.valSize = sizeZigzag64 case "bytes", "group": p.WireType = WireBytes // no numeric converter for non-numeric types @@ -299,6 +224,7 @@ func (p *Properties) Parse(s string) { return } +outer: for i := 2; i < len(fields); i++ { f := fields[i] switch { @@ -326,229 +252,28 @@ func (p *Properties) Parse(s string) { if i+1 < len(fields) { // Commas aren't escaped, and def is always last. p.Default += "," + strings.Join(fields[i+1:], ",") - break + break outer } } } } -func logNoSliceEnc(t1, t2 reflect.Type) { - fmt.Fprintf(os.Stderr, "proto: no slice oenc for %T = []%T\n", t1, t2) -} - var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() -// Initialize the fields for encoding and decoding. -func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { - p.enc = nil - p.dec = nil - p.size = nil - +// setFieldProps initializes the field properties for submessages and maps. +func (p *Properties) setFieldProps(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { switch t1 := typ; t1.Kind() { - default: - fmt.Fprintf(os.Stderr, "proto: no coders for %v\n", t1) - - // proto3 scalar types - - case reflect.Bool: - p.enc = (*Buffer).enc_proto3_bool - p.dec = (*Buffer).dec_proto3_bool - p.size = size_proto3_bool - case reflect.Int32: - p.enc = (*Buffer).enc_proto3_int32 - p.dec = (*Buffer).dec_proto3_int32 - p.size = size_proto3_int32 - case reflect.Uint32: - p.enc = (*Buffer).enc_proto3_uint32 - p.dec = (*Buffer).dec_proto3_int32 // can reuse - p.size = size_proto3_uint32 - case reflect.Int64, reflect.Uint64: - p.enc = (*Buffer).enc_proto3_int64 - p.dec = (*Buffer).dec_proto3_int64 - p.size = size_proto3_int64 - case reflect.Float32: - p.enc = (*Buffer).enc_proto3_uint32 // can just treat them as bits - p.dec = (*Buffer).dec_proto3_int32 - p.size = size_proto3_uint32 - case reflect.Float64: - p.enc = (*Buffer).enc_proto3_int64 // can just treat them as bits - p.dec = (*Buffer).dec_proto3_int64 - p.size = size_proto3_int64 - case reflect.String: - p.enc = (*Buffer).enc_proto3_string - p.dec = (*Buffer).dec_proto3_string - p.size = size_proto3_string - case reflect.Ptr: - switch t2 := t1.Elem(); t2.Kind() { - default: - fmt.Fprintf(os.Stderr, "proto: no encoder function for %v -> %v\n", t1, t2) - break - case reflect.Bool: - p.enc = (*Buffer).enc_bool - p.dec = (*Buffer).dec_bool - p.size = size_bool - case reflect.Int32: - p.enc = (*Buffer).enc_int32 - p.dec = (*Buffer).dec_int32 - p.size = size_int32 - case reflect.Uint32: - p.enc = (*Buffer).enc_uint32 - p.dec = (*Buffer).dec_int32 // can reuse - p.size = size_uint32 - case reflect.Int64, reflect.Uint64: - p.enc = (*Buffer).enc_int64 - p.dec = (*Buffer).dec_int64 - p.size = size_int64 - case reflect.Float32: - p.enc = (*Buffer).enc_uint32 // can just treat them as bits - p.dec = (*Buffer).dec_int32 - p.size = size_uint32 - case reflect.Float64: - p.enc = (*Buffer).enc_int64 // can just treat them as bits - p.dec = (*Buffer).dec_int64 - p.size = size_int64 - case reflect.String: - p.enc = (*Buffer).enc_string - p.dec = (*Buffer).dec_string - p.size = size_string - case reflect.Struct: + if t1.Elem().Kind() == reflect.Struct { p.stype = t1.Elem() - p.isMarshaler = isMarshaler(t1) - p.isUnmarshaler = isUnmarshaler(t1) - if p.Wire == "bytes" { - p.enc = (*Buffer).enc_struct_message - p.dec = (*Buffer).dec_struct_message - p.size = size_struct_message - } else { - p.enc = (*Buffer).enc_struct_group - p.dec = (*Buffer).dec_struct_group - p.size = size_struct_group - } } case reflect.Slice: - switch t2 := t1.Elem(); t2.Kind() { - default: - logNoSliceEnc(t1, t2) - break - case reflect.Bool: - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_bool - p.size = size_slice_packed_bool - } else { - p.enc = (*Buffer).enc_slice_bool - p.size = size_slice_bool - } - p.dec = (*Buffer).dec_slice_bool - p.packedDec = (*Buffer).dec_slice_packed_bool - case reflect.Int32: - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_int32 - p.size = size_slice_packed_int32 - } else { - p.enc = (*Buffer).enc_slice_int32 - p.size = size_slice_int32 - } - p.dec = (*Buffer).dec_slice_int32 - p.packedDec = (*Buffer).dec_slice_packed_int32 - case reflect.Uint32: - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_uint32 - p.size = size_slice_packed_uint32 - } else { - p.enc = (*Buffer).enc_slice_uint32 - p.size = size_slice_uint32 - } - p.dec = (*Buffer).dec_slice_int32 - p.packedDec = (*Buffer).dec_slice_packed_int32 - case reflect.Int64, reflect.Uint64: - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_int64 - p.size = size_slice_packed_int64 - } else { - p.enc = (*Buffer).enc_slice_int64 - p.size = size_slice_int64 - } - p.dec = (*Buffer).dec_slice_int64 - p.packedDec = (*Buffer).dec_slice_packed_int64 - case reflect.Uint8: - p.dec = (*Buffer).dec_slice_byte - if p.proto3 { - p.enc = (*Buffer).enc_proto3_slice_byte - p.size = size_proto3_slice_byte - } else { - p.enc = (*Buffer).enc_slice_byte - p.size = size_slice_byte - } - case reflect.Float32, reflect.Float64: - switch t2.Bits() { - case 32: - // can just treat them as bits - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_uint32 - p.size = size_slice_packed_uint32 - } else { - p.enc = (*Buffer).enc_slice_uint32 - p.size = size_slice_uint32 - } - p.dec = (*Buffer).dec_slice_int32 - p.packedDec = (*Buffer).dec_slice_packed_int32 - case 64: - // can just treat them as bits - if p.Packed { - p.enc = (*Buffer).enc_slice_packed_int64 - p.size = size_slice_packed_int64 - } else { - p.enc = (*Buffer).enc_slice_int64 - p.size = size_slice_int64 - } - p.dec = (*Buffer).dec_slice_int64 - p.packedDec = (*Buffer).dec_slice_packed_int64 - default: - logNoSliceEnc(t1, t2) - break - } - case reflect.String: - p.enc = (*Buffer).enc_slice_string - p.dec = (*Buffer).dec_slice_string - p.size = size_slice_string - case reflect.Ptr: - switch t3 := t2.Elem(); t3.Kind() { - default: - fmt.Fprintf(os.Stderr, "proto: no ptr oenc for %T -> %T -> %T\n", t1, t2, t3) - break - case reflect.Struct: - p.stype = t2.Elem() - p.isMarshaler = isMarshaler(t2) - p.isUnmarshaler = isUnmarshaler(t2) - if p.Wire == "bytes" { - p.enc = (*Buffer).enc_slice_struct_message - p.dec = (*Buffer).dec_slice_struct_message - p.size = size_slice_struct_message - } else { - p.enc = (*Buffer).enc_slice_struct_group - p.dec = (*Buffer).dec_slice_struct_group - p.size = size_slice_struct_group - } - } - case reflect.Slice: - switch t2.Elem().Kind() { - default: - fmt.Fprintf(os.Stderr, "proto: no slice elem oenc for %T -> %T -> %T\n", t1, t2, t2.Elem()) - break - case reflect.Uint8: - p.enc = (*Buffer).enc_slice_slice_byte - p.dec = (*Buffer).dec_slice_slice_byte - p.size = size_slice_slice_byte - } + if t2 := t1.Elem(); t2.Kind() == reflect.Ptr && t2.Elem().Kind() == reflect.Struct { + p.stype = t2.Elem() } case reflect.Map: - p.enc = (*Buffer).enc_new_map - p.dec = (*Buffer).dec_new_map - p.size = size_new_map - p.mtype = t1 p.mkeyprop = &Properties{} p.mkeyprop.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp) @@ -562,20 +287,6 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock p.mvalprop.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) } - // precalculate tag code - wire := p.WireType - if p.Packed { - wire = WireBytes - } - x := uint32(p.Tag)<<3 | uint32(wire) - i := 0 - for i = 0; x > 127; i++ { - p.tagbuf[i] = 0x80 | uint8(x&0x7F) - x >>= 7 - } - p.tagbuf[i] = uint8(x) - p.tagcode = p.tagbuf[0 : i+1] - if p.stype != nil { if lockGetProp { p.sprop = GetProperties(p.stype) @@ -586,32 +297,9 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock } var ( - marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() - unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() + marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() ) -// isMarshaler reports whether type t implements Marshaler. -func isMarshaler(t reflect.Type) bool { - // We're checking for (likely) pointer-receiver methods - // so if t is not a pointer, something is very wrong. - // The calls above only invoke isMarshaler on pointer types. - if t.Kind() != reflect.Ptr { - panic("proto: misuse of isMarshaler") - } - return t.Implements(marshalerType) -} - -// isUnmarshaler reports whether type t implements Unmarshaler. -func isUnmarshaler(t reflect.Type) bool { - // We're checking for (likely) pointer-receiver methods - // so if t is not a pointer, something is very wrong. - // The calls above only invoke isUnmarshaler on pointer types. - if t.Kind() != reflect.Ptr { - panic("proto: misuse of isUnmarshaler") - } - return t.Implements(unmarshalerType) -} - // Init populates the properties from a protocol buffer struct tag. func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) { p.init(typ, name, tag, f, true) @@ -621,14 +309,11 @@ func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructF // "bytes,49,opt,def=hello!" p.Name = name p.OrigName = name - if f != nil { - p.field = toField(f) - } if tag == "" { return } p.Parse(tag) - p.setEncAndDec(typ, f, lockGetProp) + p.setFieldProps(typ, f, lockGetProp) } var ( @@ -678,9 +363,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { propertiesMap[t] = prop // build properties - prop.extendable = reflect.PtrTo(t).Implements(extendableProtoType) || - reflect.PtrTo(t).Implements(extendableProtoV1Type) - prop.unrecField = invalidField prop.Prop = make([]*Properties, t.NumField()) prop.order = make([]int, t.NumField()) @@ -690,17 +372,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { name := f.Name p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) - if f.Name == "XXX_InternalExtensions" { // special case - p.enc = (*Buffer).enc_exts - p.dec = nil // not needed - p.size = size_exts - } else if f.Name == "XXX_extensions" { // special case - p.enc = (*Buffer).enc_map - p.dec = nil // not needed - p.size = size_map - } else if f.Name == "XXX_unrecognized" { // special case - prop.unrecField = toField(&f) - } oneof := f.Tag.Get("protobuf_oneof") // special case if oneof != "" { // Oneof fields don't use the traditional protobuf tag. @@ -715,9 +386,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { } print("\n") } - if p.enc == nil && !strings.HasPrefix(f.Name, "XXX_") && oneof == "" { - fmt.Fprintln(os.Stderr, "proto: no encoder for", f.Name, f.Type.String(), "[GetProperties]") - } } // Re-order prop.order. @@ -728,8 +396,7 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { } if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok { var oots []interface{} - prop.oneofMarshaler, prop.oneofUnmarshaler, prop.oneofSizer, oots = om.XXX_OneofFuncs() - prop.stype = t + _, _, _, oots = om.XXX_OneofFuncs() // Interpret oneof metadata. prop.OneofTypes = make(map[string]*OneofProperties) @@ -779,30 +446,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties { return prop } -// Return the Properties object for the x[0]'th field of the structure. -func propByIndex(t reflect.Type, x []int) *Properties { - if len(x) != 1 { - fmt.Fprintf(os.Stderr, "proto: field index dimension %d (not 1) for type %s\n", len(x), t) - return nil - } - prop := GetProperties(t) - return prop.Prop[x[0]] -} - -// Get the address and type of a pointer to a struct from an interface. -func getbase(pb Message) (t reflect.Type, b structPointer, err error) { - if pb == nil { - err = ErrNil - return - } - // get the reflect type of the pointer to the struct. - t = reflect.TypeOf(pb) - // get the address of the struct. - value := reflect.ValueOf(pb) - b = toStructPointer(value) - return -} - // A global registry of enum types. // The generated code will register the generated maps by calling RegisterEnum. @@ -826,20 +469,42 @@ func EnumValueMap(enumType string) map[string]int32 { // A registry of all linked message types. // The string is a fully-qualified proto name ("pkg.Message"). var ( - protoTypes = make(map[string]reflect.Type) - revProtoTypes = make(map[reflect.Type]string) + protoTypedNils = make(map[string]Message) // a map from proto names to typed nil pointers + protoMapTypes = make(map[string]reflect.Type) // a map from proto names to map types + revProtoTypes = make(map[reflect.Type]string) ) // RegisterType is called from generated code and maps from the fully qualified // proto name to the type (pointer to struct) of the protocol buffer. func RegisterType(x Message, name string) { - if _, ok := protoTypes[name]; ok { + if _, ok := protoTypedNils[name]; ok { // TODO: Some day, make this a panic. log.Printf("proto: duplicate proto type registered: %s", name) return } t := reflect.TypeOf(x) - protoTypes[name] = t + if v := reflect.ValueOf(x); v.Kind() == reflect.Ptr && v.Pointer() == 0 { + // Generated code always calls RegisterType with nil x. + // This check is just for extra safety. + protoTypedNils[name] = x + } else { + protoTypedNils[name] = reflect.Zero(t).Interface().(Message) + } + revProtoTypes[t] = name +} + +// RegisterMapType is called from generated code and maps from the fully qualified +// proto name to the native map type of the proto map definition. +func RegisterMapType(x interface{}, name string) { + if reflect.TypeOf(x).Kind() != reflect.Map { + panic(fmt.Sprintf("RegisterMapType(%T, %q); want map", x, name)) + } + if _, ok := protoMapTypes[name]; ok { + log.Printf("proto: duplicate proto type registered: %s", name) + return + } + t := reflect.TypeOf(x) + protoMapTypes[name] = t revProtoTypes[t] = name } @@ -855,7 +520,14 @@ func MessageName(x Message) string { } // MessageType returns the message type (pointer to struct) for a named message. -func MessageType(name string) reflect.Type { return protoTypes[name] } +// The type is not guaranteed to implement proto.Message if the name refers to a +// map entry. +func MessageType(name string) reflect.Type { + if t, ok := protoTypedNils[name]; ok { + return reflect.TypeOf(t) + } + return protoMapTypes[name] +} // A registry of all linked proto files. var ( diff --git a/vendor/github.com/golang/protobuf/proto/table_marshal.go b/vendor/github.com/golang/protobuf/proto/table_marshal.go new file mode 100644 index 00000000..0f212b30 --- /dev/null +++ b/vendor/github.com/golang/protobuf/proto/table_marshal.go @@ -0,0 +1,2681 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "errors" + "fmt" + "math" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "unicode/utf8" +) + +// a sizer takes a pointer to a field and the size of its tag, computes the size of +// the encoded data. +type sizer func(pointer, int) int + +// a marshaler takes a byte slice, a pointer to a field, and its tag (in wire format), +// marshals the field to the end of the slice, returns the slice and error (if any). +type marshaler func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) + +// marshalInfo is the information used for marshaling a message. +type marshalInfo struct { + typ reflect.Type + fields []*marshalFieldInfo + unrecognized field // offset of XXX_unrecognized + extensions field // offset of XXX_InternalExtensions + v1extensions field // offset of XXX_extensions + sizecache field // offset of XXX_sizecache + initialized int32 // 0 -- only typ is set, 1 -- fully initialized + messageset bool // uses message set wire format + hasmarshaler bool // has custom marshaler + sync.RWMutex // protect extElems map, also for initialization + extElems map[int32]*marshalElemInfo // info of extension elements +} + +// marshalFieldInfo is the information used for marshaling a field of a message. +type marshalFieldInfo struct { + field field + wiretag uint64 // tag in wire format + tagsize int // size of tag in wire format + sizer sizer + marshaler marshaler + isPointer bool + required bool // field is required + name string // name of the field, for error reporting + oneofElems map[reflect.Type]*marshalElemInfo // info of oneof elements +} + +// marshalElemInfo is the information used for marshaling an extension or oneof element. +type marshalElemInfo struct { + wiretag uint64 // tag in wire format + tagsize int // size of tag in wire format + sizer sizer + marshaler marshaler + isptr bool // elem is pointer typed, thus interface of this type is a direct interface (extension only) +} + +var ( + marshalInfoMap = map[reflect.Type]*marshalInfo{} + marshalInfoLock sync.Mutex +) + +// getMarshalInfo returns the information to marshal a given type of message. +// The info it returns may not necessarily initialized. +// t is the type of the message (NOT the pointer to it). +func getMarshalInfo(t reflect.Type) *marshalInfo { + marshalInfoLock.Lock() + u, ok := marshalInfoMap[t] + if !ok { + u = &marshalInfo{typ: t} + marshalInfoMap[t] = u + } + marshalInfoLock.Unlock() + return u +} + +// Size is the entry point from generated code, +// and should be ONLY called by generated code. +// It computes the size of encoded data of msg. +// a is a pointer to a place to store cached marshal info. +func (a *InternalMessageInfo) Size(msg Message) int { + u := getMessageMarshalInfo(msg, a) + ptr := toPointer(&msg) + if ptr.isNil() { + // We get here if msg is a typed nil ((*SomeMessage)(nil)), + // so it satisfies the interface, and msg == nil wouldn't + // catch it. We don't want crash in this case. + return 0 + } + return u.size(ptr) +} + +// Marshal is the entry point from generated code, +// and should be ONLY called by generated code. +// It marshals msg to the end of b. +// a is a pointer to a place to store cached marshal info. +func (a *InternalMessageInfo) Marshal(b []byte, msg Message, deterministic bool) ([]byte, error) { + u := getMessageMarshalInfo(msg, a) + ptr := toPointer(&msg) + if ptr.isNil() { + // We get here if msg is a typed nil ((*SomeMessage)(nil)), + // so it satisfies the interface, and msg == nil wouldn't + // catch it. We don't want crash in this case. + return b, ErrNil + } + return u.marshal(b, ptr, deterministic) +} + +func getMessageMarshalInfo(msg interface{}, a *InternalMessageInfo) *marshalInfo { + // u := a.marshal, but atomically. + // We use an atomic here to ensure memory consistency. + u := atomicLoadMarshalInfo(&a.marshal) + if u == nil { + // Get marshal information from type of message. + t := reflect.ValueOf(msg).Type() + if t.Kind() != reflect.Ptr { + panic(fmt.Sprintf("cannot handle non-pointer message type %v", t)) + } + u = getMarshalInfo(t.Elem()) + // Store it in the cache for later users. + // a.marshal = u, but atomically. + atomicStoreMarshalInfo(&a.marshal, u) + } + return u +} + +// size is the main function to compute the size of the encoded data of a message. +// ptr is the pointer to the message. +func (u *marshalInfo) size(ptr pointer) int { + if atomic.LoadInt32(&u.initialized) == 0 { + u.computeMarshalInfo() + } + + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + if u.hasmarshaler { + m := ptr.asPointerTo(u.typ).Interface().(Marshaler) + b, _ := m.Marshal() + return len(b) + } + + n := 0 + for _, f := range u.fields { + if f.isPointer && ptr.offset(f.field).getPointer().isNil() { + // nil pointer always marshals to nothing + continue + } + n += f.sizer(ptr.offset(f.field), f.tagsize) + } + if u.extensions.IsValid() { + e := ptr.offset(u.extensions).toExtensions() + if u.messageset { + n += u.sizeMessageSet(e) + } else { + n += u.sizeExtensions(e) + } + } + if u.v1extensions.IsValid() { + m := *ptr.offset(u.v1extensions).toOldExtensions() + n += u.sizeV1Extensions(m) + } + if u.unrecognized.IsValid() { + s := *ptr.offset(u.unrecognized).toBytes() + n += len(s) + } + // cache the result for use in marshal + if u.sizecache.IsValid() { + atomic.StoreInt32(ptr.offset(u.sizecache).toInt32(), int32(n)) + } + return n +} + +// cachedsize gets the size from cache. If there is no cache (i.e. message is not generated), +// fall back to compute the size. +func (u *marshalInfo) cachedsize(ptr pointer) int { + if u.sizecache.IsValid() { + return int(atomic.LoadInt32(ptr.offset(u.sizecache).toInt32())) + } + return u.size(ptr) +} + +// marshal is the main function to marshal a message. It takes a byte slice and appends +// the encoded data to the end of the slice, returns the slice and error (if any). +// ptr is the pointer to the message. +// If deterministic is true, map is marshaled in deterministic order. +func (u *marshalInfo) marshal(b []byte, ptr pointer, deterministic bool) ([]byte, error) { + if atomic.LoadInt32(&u.initialized) == 0 { + u.computeMarshalInfo() + } + + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + if u.hasmarshaler { + m := ptr.asPointerTo(u.typ).Interface().(Marshaler) + b1, err := m.Marshal() + b = append(b, b1...) + return b, err + } + + var err, errreq error + // The old marshaler encodes extensions at beginning. + if u.extensions.IsValid() { + e := ptr.offset(u.extensions).toExtensions() + if u.messageset { + b, err = u.appendMessageSet(b, e, deterministic) + } else { + b, err = u.appendExtensions(b, e, deterministic) + } + if err != nil { + return b, err + } + } + if u.v1extensions.IsValid() { + m := *ptr.offset(u.v1extensions).toOldExtensions() + b, err = u.appendV1Extensions(b, m, deterministic) + if err != nil { + return b, err + } + } + for _, f := range u.fields { + if f.required && errreq == nil { + if ptr.offset(f.field).getPointer().isNil() { + // Required field is not set. + // We record the error but keep going, to give a complete marshaling. + errreq = &RequiredNotSetError{f.name} + continue + } + } + if f.isPointer && ptr.offset(f.field).getPointer().isNil() { + // nil pointer always marshals to nothing + continue + } + b, err = f.marshaler(b, ptr.offset(f.field), f.wiretag, deterministic) + if err != nil { + if err1, ok := err.(*RequiredNotSetError); ok { + // Required field in submessage is not set. + // We record the error but keep going, to give a complete marshaling. + if errreq == nil { + errreq = &RequiredNotSetError{f.name + "." + err1.field} + } + continue + } + if err == errRepeatedHasNil { + err = errors.New("proto: repeated field " + f.name + " has nil element") + } + return b, err + } + } + if u.unrecognized.IsValid() { + s := *ptr.offset(u.unrecognized).toBytes() + b = append(b, s...) + } + return b, errreq +} + +// computeMarshalInfo initializes the marshal info. +func (u *marshalInfo) computeMarshalInfo() { + u.Lock() + defer u.Unlock() + if u.initialized != 0 { // non-atomic read is ok as it is protected by the lock + return + } + + t := u.typ + u.unrecognized = invalidField + u.extensions = invalidField + u.v1extensions = invalidField + u.sizecache = invalidField + + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + if reflect.PtrTo(t).Implements(marshalerType) { + u.hasmarshaler = true + atomic.StoreInt32(&u.initialized, 1) + return + } + + // get oneof implementers + var oneofImplementers []interface{} + if m, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok { + _, _, _, oneofImplementers = m.XXX_OneofFuncs() + } + + n := t.NumField() + + // deal with XXX fields first + for i := 0; i < t.NumField(); i++ { + f := t.Field(i) + if !strings.HasPrefix(f.Name, "XXX_") { + continue + } + switch f.Name { + case "XXX_sizecache": + u.sizecache = toField(&f) + case "XXX_unrecognized": + u.unrecognized = toField(&f) + case "XXX_InternalExtensions": + u.extensions = toField(&f) + u.messageset = f.Tag.Get("protobuf_messageset") == "1" + case "XXX_extensions": + u.v1extensions = toField(&f) + case "XXX_NoUnkeyedLiteral": + // nothing to do + default: + panic("unknown XXX field: " + f.Name) + } + n-- + } + + // normal fields + fields := make([]marshalFieldInfo, n) // batch allocation + u.fields = make([]*marshalFieldInfo, 0, n) + for i, j := 0, 0; i < t.NumField(); i++ { + f := t.Field(i) + + if strings.HasPrefix(f.Name, "XXX_") { + continue + } + field := &fields[j] + j++ + field.name = f.Name + u.fields = append(u.fields, field) + if f.Tag.Get("protobuf_oneof") != "" { + field.computeOneofFieldInfo(&f, oneofImplementers) + continue + } + if f.Tag.Get("protobuf") == "" { + // field has no tag (not in generated message), ignore it + u.fields = u.fields[:len(u.fields)-1] + j-- + continue + } + field.computeMarshalFieldInfo(&f) + } + + // fields are marshaled in tag order on the wire. + sort.Sort(byTag(u.fields)) + + atomic.StoreInt32(&u.initialized, 1) +} + +// helper for sorting fields by tag +type byTag []*marshalFieldInfo + +func (a byTag) Len() int { return len(a) } +func (a byTag) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byTag) Less(i, j int) bool { return a[i].wiretag < a[j].wiretag } + +// getExtElemInfo returns the information to marshal an extension element. +// The info it returns is initialized. +func (u *marshalInfo) getExtElemInfo(desc *ExtensionDesc) *marshalElemInfo { + // get from cache first + u.RLock() + e, ok := u.extElems[desc.Field] + u.RUnlock() + if ok { + return e + } + + t := reflect.TypeOf(desc.ExtensionType) // pointer or slice to basic type or struct + tags := strings.Split(desc.Tag, ",") + tag, err := strconv.Atoi(tags[1]) + if err != nil { + panic("tag is not an integer") + } + wt := wiretype(tags[0]) + sizer, marshaler := typeMarshaler(t, tags, false, false) + e = &marshalElemInfo{ + wiretag: uint64(tag)<<3 | wt, + tagsize: SizeVarint(uint64(tag) << 3), + sizer: sizer, + marshaler: marshaler, + isptr: t.Kind() == reflect.Ptr, + } + + // update cache + u.Lock() + if u.extElems == nil { + u.extElems = make(map[int32]*marshalElemInfo) + } + u.extElems[desc.Field] = e + u.Unlock() + return e +} + +// computeMarshalFieldInfo fills up the information to marshal a field. +func (fi *marshalFieldInfo) computeMarshalFieldInfo(f *reflect.StructField) { + // parse protobuf tag of the field. + // tag has format of "bytes,49,opt,name=foo,def=hello!" + tags := strings.Split(f.Tag.Get("protobuf"), ",") + if tags[0] == "" { + return + } + tag, err := strconv.Atoi(tags[1]) + if err != nil { + panic("tag is not an integer") + } + wt := wiretype(tags[0]) + if tags[2] == "req" { + fi.required = true + } + fi.setTag(f, tag, wt) + fi.setMarshaler(f, tags) +} + +func (fi *marshalFieldInfo) computeOneofFieldInfo(f *reflect.StructField, oneofImplementers []interface{}) { + fi.field = toField(f) + fi.wiretag = 1<<31 - 1 // Use a large tag number, make oneofs sorted at the end. This tag will not appear on the wire. + fi.isPointer = true + fi.sizer, fi.marshaler = makeOneOfMarshaler(fi, f) + fi.oneofElems = make(map[reflect.Type]*marshalElemInfo) + + ityp := f.Type // interface type + for _, o := range oneofImplementers { + t := reflect.TypeOf(o) + if !t.Implements(ityp) { + continue + } + sf := t.Elem().Field(0) // oneof implementer is a struct with a single field + tags := strings.Split(sf.Tag.Get("protobuf"), ",") + tag, err := strconv.Atoi(tags[1]) + if err != nil { + panic("tag is not an integer") + } + wt := wiretype(tags[0]) + sizer, marshaler := typeMarshaler(sf.Type, tags, false, true) // oneof should not omit any zero value + fi.oneofElems[t.Elem()] = &marshalElemInfo{ + wiretag: uint64(tag)<<3 | wt, + tagsize: SizeVarint(uint64(tag) << 3), + sizer: sizer, + marshaler: marshaler, + } + } +} + +type oneofMessage interface { + XXX_OneofFuncs() (func(Message, *Buffer) error, func(Message, int, int, *Buffer) (bool, error), func(Message) int, []interface{}) +} + +// wiretype returns the wire encoding of the type. +func wiretype(encoding string) uint64 { + switch encoding { + case "fixed32": + return WireFixed32 + case "fixed64": + return WireFixed64 + case "varint", "zigzag32", "zigzag64": + return WireVarint + case "bytes": + return WireBytes + case "group": + return WireStartGroup + } + panic("unknown wire type " + encoding) +} + +// setTag fills up the tag (in wire format) and its size in the info of a field. +func (fi *marshalFieldInfo) setTag(f *reflect.StructField, tag int, wt uint64) { + fi.field = toField(f) + fi.wiretag = uint64(tag)<<3 | wt + fi.tagsize = SizeVarint(uint64(tag) << 3) +} + +// setMarshaler fills up the sizer and marshaler in the info of a field. +func (fi *marshalFieldInfo) setMarshaler(f *reflect.StructField, tags []string) { + switch f.Type.Kind() { + case reflect.Map: + // map field + fi.isPointer = true + fi.sizer, fi.marshaler = makeMapMarshaler(f) + return + case reflect.Ptr, reflect.Slice: + fi.isPointer = true + } + fi.sizer, fi.marshaler = typeMarshaler(f.Type, tags, true, false) +} + +// typeMarshaler returns the sizer and marshaler of a given field. +// t is the type of the field. +// tags is the generated "protobuf" tag of the field. +// If nozero is true, zero value is not marshaled to the wire. +// If oneof is true, it is a oneof field. +func typeMarshaler(t reflect.Type, tags []string, nozero, oneof bool) (sizer, marshaler) { + encoding := tags[0] + + pointer := false + slice := false + if t.Kind() == reflect.Slice && t.Elem().Kind() != reflect.Uint8 { + slice = true + t = t.Elem() + } + if t.Kind() == reflect.Ptr { + pointer = true + t = t.Elem() + } + + packed := false + proto3 := false + for i := 2; i < len(tags); i++ { + if tags[i] == "packed" { + packed = true + } + if tags[i] == "proto3" { + proto3 = true + } + } + + switch t.Kind() { + case reflect.Bool: + if pointer { + return sizeBoolPtr, appendBoolPtr + } + if slice { + if packed { + return sizeBoolPackedSlice, appendBoolPackedSlice + } + return sizeBoolSlice, appendBoolSlice + } + if nozero { + return sizeBoolValueNoZero, appendBoolValueNoZero + } + return sizeBoolValue, appendBoolValue + case reflect.Uint32: + switch encoding { + case "fixed32": + if pointer { + return sizeFixed32Ptr, appendFixed32Ptr + } + if slice { + if packed { + return sizeFixed32PackedSlice, appendFixed32PackedSlice + } + return sizeFixed32Slice, appendFixed32Slice + } + if nozero { + return sizeFixed32ValueNoZero, appendFixed32ValueNoZero + } + return sizeFixed32Value, appendFixed32Value + case "varint": + if pointer { + return sizeVarint32Ptr, appendVarint32Ptr + } + if slice { + if packed { + return sizeVarint32PackedSlice, appendVarint32PackedSlice + } + return sizeVarint32Slice, appendVarint32Slice + } + if nozero { + return sizeVarint32ValueNoZero, appendVarint32ValueNoZero + } + return sizeVarint32Value, appendVarint32Value + } + case reflect.Int32: + switch encoding { + case "fixed32": + if pointer { + return sizeFixedS32Ptr, appendFixedS32Ptr + } + if slice { + if packed { + return sizeFixedS32PackedSlice, appendFixedS32PackedSlice + } + return sizeFixedS32Slice, appendFixedS32Slice + } + if nozero { + return sizeFixedS32ValueNoZero, appendFixedS32ValueNoZero + } + return sizeFixedS32Value, appendFixedS32Value + case "varint": + if pointer { + return sizeVarintS32Ptr, appendVarintS32Ptr + } + if slice { + if packed { + return sizeVarintS32PackedSlice, appendVarintS32PackedSlice + } + return sizeVarintS32Slice, appendVarintS32Slice + } + if nozero { + return sizeVarintS32ValueNoZero, appendVarintS32ValueNoZero + } + return sizeVarintS32Value, appendVarintS32Value + case "zigzag32": + if pointer { + return sizeZigzag32Ptr, appendZigzag32Ptr + } + if slice { + if packed { + return sizeZigzag32PackedSlice, appendZigzag32PackedSlice + } + return sizeZigzag32Slice, appendZigzag32Slice + } + if nozero { + return sizeZigzag32ValueNoZero, appendZigzag32ValueNoZero + } + return sizeZigzag32Value, appendZigzag32Value + } + case reflect.Uint64: + switch encoding { + case "fixed64": + if pointer { + return sizeFixed64Ptr, appendFixed64Ptr + } + if slice { + if packed { + return sizeFixed64PackedSlice, appendFixed64PackedSlice + } + return sizeFixed64Slice, appendFixed64Slice + } + if nozero { + return sizeFixed64ValueNoZero, appendFixed64ValueNoZero + } + return sizeFixed64Value, appendFixed64Value + case "varint": + if pointer { + return sizeVarint64Ptr, appendVarint64Ptr + } + if slice { + if packed { + return sizeVarint64PackedSlice, appendVarint64PackedSlice + } + return sizeVarint64Slice, appendVarint64Slice + } + if nozero { + return sizeVarint64ValueNoZero, appendVarint64ValueNoZero + } + return sizeVarint64Value, appendVarint64Value + } + case reflect.Int64: + switch encoding { + case "fixed64": + if pointer { + return sizeFixedS64Ptr, appendFixedS64Ptr + } + if slice { + if packed { + return sizeFixedS64PackedSlice, appendFixedS64PackedSlice + } + return sizeFixedS64Slice, appendFixedS64Slice + } + if nozero { + return sizeFixedS64ValueNoZero, appendFixedS64ValueNoZero + } + return sizeFixedS64Value, appendFixedS64Value + case "varint": + if pointer { + return sizeVarintS64Ptr, appendVarintS64Ptr + } + if slice { + if packed { + return sizeVarintS64PackedSlice, appendVarintS64PackedSlice + } + return sizeVarintS64Slice, appendVarintS64Slice + } + if nozero { + return sizeVarintS64ValueNoZero, appendVarintS64ValueNoZero + } + return sizeVarintS64Value, appendVarintS64Value + case "zigzag64": + if pointer { + return sizeZigzag64Ptr, appendZigzag64Ptr + } + if slice { + if packed { + return sizeZigzag64PackedSlice, appendZigzag64PackedSlice + } + return sizeZigzag64Slice, appendZigzag64Slice + } + if nozero { + return sizeZigzag64ValueNoZero, appendZigzag64ValueNoZero + } + return sizeZigzag64Value, appendZigzag64Value + } + case reflect.Float32: + if pointer { + return sizeFloat32Ptr, appendFloat32Ptr + } + if slice { + if packed { + return sizeFloat32PackedSlice, appendFloat32PackedSlice + } + return sizeFloat32Slice, appendFloat32Slice + } + if nozero { + return sizeFloat32ValueNoZero, appendFloat32ValueNoZero + } + return sizeFloat32Value, appendFloat32Value + case reflect.Float64: + if pointer { + return sizeFloat64Ptr, appendFloat64Ptr + } + if slice { + if packed { + return sizeFloat64PackedSlice, appendFloat64PackedSlice + } + return sizeFloat64Slice, appendFloat64Slice + } + if nozero { + return sizeFloat64ValueNoZero, appendFloat64ValueNoZero + } + return sizeFloat64Value, appendFloat64Value + case reflect.String: + if pointer { + return sizeStringPtr, appendStringPtr + } + if slice { + return sizeStringSlice, appendStringSlice + } + if nozero { + return sizeStringValueNoZero, appendStringValueNoZero + } + return sizeStringValue, appendStringValue + case reflect.Slice: + if slice { + return sizeBytesSlice, appendBytesSlice + } + if oneof { + // Oneof bytes field may also have "proto3" tag. + // We want to marshal it as a oneof field. Do this + // check before the proto3 check. + return sizeBytesOneof, appendBytesOneof + } + if proto3 { + return sizeBytes3, appendBytes3 + } + return sizeBytes, appendBytes + case reflect.Struct: + switch encoding { + case "group": + if slice { + return makeGroupSliceMarshaler(getMarshalInfo(t)) + } + return makeGroupMarshaler(getMarshalInfo(t)) + case "bytes": + if slice { + return makeMessageSliceMarshaler(getMarshalInfo(t)) + } + return makeMessageMarshaler(getMarshalInfo(t)) + } + } + panic(fmt.Sprintf("unknown or mismatched type: type: %v, wire type: %v", t, encoding)) +} + +// Below are functions to size/marshal a specific type of a field. +// They are stored in the field's info, and called by function pointers. +// They have type sizer or marshaler. + +func sizeFixed32Value(_ pointer, tagsize int) int { + return 4 + tagsize +} +func sizeFixed32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toUint32() + if v == 0 { + return 0 + } + return 4 + tagsize +} +func sizeFixed32Ptr(ptr pointer, tagsize int) int { + p := *ptr.toUint32Ptr() + if p == nil { + return 0 + } + return 4 + tagsize +} +func sizeFixed32Slice(ptr pointer, tagsize int) int { + s := *ptr.toUint32Slice() + return (4 + tagsize) * len(s) +} +func sizeFixed32PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toUint32Slice() + if len(s) == 0 { + return 0 + } + return 4*len(s) + SizeVarint(uint64(4*len(s))) + tagsize +} +func sizeFixedS32Value(_ pointer, tagsize int) int { + return 4 + tagsize +} +func sizeFixedS32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + if v == 0 { + return 0 + } + return 4 + tagsize +} +func sizeFixedS32Ptr(ptr pointer, tagsize int) int { + p := ptr.getInt32Ptr() + if p == nil { + return 0 + } + return 4 + tagsize +} +func sizeFixedS32Slice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + return (4 + tagsize) * len(s) +} +func sizeFixedS32PackedSlice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + if len(s) == 0 { + return 0 + } + return 4*len(s) + SizeVarint(uint64(4*len(s))) + tagsize +} +func sizeFloat32Value(_ pointer, tagsize int) int { + return 4 + tagsize +} +func sizeFloat32ValueNoZero(ptr pointer, tagsize int) int { + v := math.Float32bits(*ptr.toFloat32()) + if v == 0 { + return 0 + } + return 4 + tagsize +} +func sizeFloat32Ptr(ptr pointer, tagsize int) int { + p := *ptr.toFloat32Ptr() + if p == nil { + return 0 + } + return 4 + tagsize +} +func sizeFloat32Slice(ptr pointer, tagsize int) int { + s := *ptr.toFloat32Slice() + return (4 + tagsize) * len(s) +} +func sizeFloat32PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toFloat32Slice() + if len(s) == 0 { + return 0 + } + return 4*len(s) + SizeVarint(uint64(4*len(s))) + tagsize +} +func sizeFixed64Value(_ pointer, tagsize int) int { + return 8 + tagsize +} +func sizeFixed64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toUint64() + if v == 0 { + return 0 + } + return 8 + tagsize +} +func sizeFixed64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toUint64Ptr() + if p == nil { + return 0 + } + return 8 + tagsize +} +func sizeFixed64Slice(ptr pointer, tagsize int) int { + s := *ptr.toUint64Slice() + return (8 + tagsize) * len(s) +} +func sizeFixed64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toUint64Slice() + if len(s) == 0 { + return 0 + } + return 8*len(s) + SizeVarint(uint64(8*len(s))) + tagsize +} +func sizeFixedS64Value(_ pointer, tagsize int) int { + return 8 + tagsize +} +func sizeFixedS64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + if v == 0 { + return 0 + } + return 8 + tagsize +} +func sizeFixedS64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toInt64Ptr() + if p == nil { + return 0 + } + return 8 + tagsize +} +func sizeFixedS64Slice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + return (8 + tagsize) * len(s) +} +func sizeFixedS64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return 0 + } + return 8*len(s) + SizeVarint(uint64(8*len(s))) + tagsize +} +func sizeFloat64Value(_ pointer, tagsize int) int { + return 8 + tagsize +} +func sizeFloat64ValueNoZero(ptr pointer, tagsize int) int { + v := math.Float64bits(*ptr.toFloat64()) + if v == 0 { + return 0 + } + return 8 + tagsize +} +func sizeFloat64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toFloat64Ptr() + if p == nil { + return 0 + } + return 8 + tagsize +} +func sizeFloat64Slice(ptr pointer, tagsize int) int { + s := *ptr.toFloat64Slice() + return (8 + tagsize) * len(s) +} +func sizeFloat64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toFloat64Slice() + if len(s) == 0 { + return 0 + } + return 8*len(s) + SizeVarint(uint64(8*len(s))) + tagsize +} +func sizeVarint32Value(ptr pointer, tagsize int) int { + v := *ptr.toUint32() + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarint32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toUint32() + if v == 0 { + return 0 + } + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarint32Ptr(ptr pointer, tagsize int) int { + p := *ptr.toUint32Ptr() + if p == nil { + return 0 + } + return SizeVarint(uint64(*p)) + tagsize +} +func sizeVarint32Slice(ptr pointer, tagsize int) int { + s := *ptr.toUint32Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + tagsize + } + return n +} +func sizeVarint32PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toUint32Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeVarintS32Value(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarintS32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + if v == 0 { + return 0 + } + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarintS32Ptr(ptr pointer, tagsize int) int { + p := ptr.getInt32Ptr() + if p == nil { + return 0 + } + return SizeVarint(uint64(*p)) + tagsize +} +func sizeVarintS32Slice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + tagsize + } + return n +} +func sizeVarintS32PackedSlice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeVarint64Value(ptr pointer, tagsize int) int { + v := *ptr.toUint64() + return SizeVarint(v) + tagsize +} +func sizeVarint64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toUint64() + if v == 0 { + return 0 + } + return SizeVarint(v) + tagsize +} +func sizeVarint64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toUint64Ptr() + if p == nil { + return 0 + } + return SizeVarint(*p) + tagsize +} +func sizeVarint64Slice(ptr pointer, tagsize int) int { + s := *ptr.toUint64Slice() + n := 0 + for _, v := range s { + n += SizeVarint(v) + tagsize + } + return n +} +func sizeVarint64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toUint64Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(v) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeVarintS64Value(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarintS64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + if v == 0 { + return 0 + } + return SizeVarint(uint64(v)) + tagsize +} +func sizeVarintS64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toInt64Ptr() + if p == nil { + return 0 + } + return SizeVarint(uint64(*p)) + tagsize +} +func sizeVarintS64Slice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + tagsize + } + return n +} +func sizeVarintS64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeZigzag32Value(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + return SizeVarint(uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + tagsize +} +func sizeZigzag32ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt32() + if v == 0 { + return 0 + } + return SizeVarint(uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + tagsize +} +func sizeZigzag32Ptr(ptr pointer, tagsize int) int { + p := ptr.getInt32Ptr() + if p == nil { + return 0 + } + v := *p + return SizeVarint(uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + tagsize +} +func sizeZigzag32Slice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + tagsize + } + return n +} +func sizeZigzag32PackedSlice(ptr pointer, tagsize int) int { + s := ptr.getInt32Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64((uint32(v) << 1) ^ uint32((int32(v) >> 31)))) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeZigzag64Value(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + return SizeVarint(uint64(v<<1)^uint64((int64(v)>>63))) + tagsize +} +func sizeZigzag64ValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toInt64() + if v == 0 { + return 0 + } + return SizeVarint(uint64(v<<1)^uint64((int64(v)>>63))) + tagsize +} +func sizeZigzag64Ptr(ptr pointer, tagsize int) int { + p := *ptr.toInt64Ptr() + if p == nil { + return 0 + } + v := *p + return SizeVarint(uint64(v<<1)^uint64((int64(v)>>63))) + tagsize +} +func sizeZigzag64Slice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v<<1)^uint64((int64(v)>>63))) + tagsize + } + return n +} +func sizeZigzag64PackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return 0 + } + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v<<1) ^ uint64((int64(v) >> 63))) + } + return n + SizeVarint(uint64(n)) + tagsize +} +func sizeBoolValue(_ pointer, tagsize int) int { + return 1 + tagsize +} +func sizeBoolValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toBool() + if !v { + return 0 + } + return 1 + tagsize +} +func sizeBoolPtr(ptr pointer, tagsize int) int { + p := *ptr.toBoolPtr() + if p == nil { + return 0 + } + return 1 + tagsize +} +func sizeBoolSlice(ptr pointer, tagsize int) int { + s := *ptr.toBoolSlice() + return (1 + tagsize) * len(s) +} +func sizeBoolPackedSlice(ptr pointer, tagsize int) int { + s := *ptr.toBoolSlice() + if len(s) == 0 { + return 0 + } + return len(s) + SizeVarint(uint64(len(s))) + tagsize +} +func sizeStringValue(ptr pointer, tagsize int) int { + v := *ptr.toString() + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeStringValueNoZero(ptr pointer, tagsize int) int { + v := *ptr.toString() + if v == "" { + return 0 + } + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeStringPtr(ptr pointer, tagsize int) int { + p := *ptr.toStringPtr() + if p == nil { + return 0 + } + v := *p + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeStringSlice(ptr pointer, tagsize int) int { + s := *ptr.toStringSlice() + n := 0 + for _, v := range s { + n += len(v) + SizeVarint(uint64(len(v))) + tagsize + } + return n +} +func sizeBytes(ptr pointer, tagsize int) int { + v := *ptr.toBytes() + if v == nil { + return 0 + } + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeBytes3(ptr pointer, tagsize int) int { + v := *ptr.toBytes() + if len(v) == 0 { + return 0 + } + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeBytesOneof(ptr pointer, tagsize int) int { + v := *ptr.toBytes() + return len(v) + SizeVarint(uint64(len(v))) + tagsize +} +func sizeBytesSlice(ptr pointer, tagsize int) int { + s := *ptr.toBytesSlice() + n := 0 + for _, v := range s { + n += len(v) + SizeVarint(uint64(len(v))) + tagsize + } + return n +} + +// appendFixed32 appends an encoded fixed32 to b. +func appendFixed32(b []byte, v uint32) []byte { + b = append(b, + byte(v), + byte(v>>8), + byte(v>>16), + byte(v>>24)) + return b +} + +// appendFixed64 appends an encoded fixed64 to b. +func appendFixed64(b []byte, v uint64) []byte { + b = append(b, + byte(v), + byte(v>>8), + byte(v>>16), + byte(v>>24), + byte(v>>32), + byte(v>>40), + byte(v>>48), + byte(v>>56)) + return b +} + +// appendVarint appends an encoded varint to b. +func appendVarint(b []byte, v uint64) []byte { + // TODO: make 1-byte (maybe 2-byte) case inline-able, once we + // have non-leaf inliner. + switch { + case v < 1<<7: + b = append(b, byte(v)) + case v < 1<<14: + b = append(b, + byte(v&0x7f|0x80), + byte(v>>7)) + case v < 1<<21: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte(v>>14)) + case v < 1<<28: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte(v>>21)) + case v < 1<<35: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte(v>>28)) + case v < 1<<42: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte(v>>35)) + case v < 1<<49: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte(v>>42)) + case v < 1<<56: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte(v>>49)) + case v < 1<<63: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte((v>>49)&0x7f|0x80), + byte(v>>56)) + default: + b = append(b, + byte(v&0x7f|0x80), + byte((v>>7)&0x7f|0x80), + byte((v>>14)&0x7f|0x80), + byte((v>>21)&0x7f|0x80), + byte((v>>28)&0x7f|0x80), + byte((v>>35)&0x7f|0x80), + byte((v>>42)&0x7f|0x80), + byte((v>>49)&0x7f|0x80), + byte((v>>56)&0x7f|0x80), + 1) + } + return b +} + +func appendFixed32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint32() + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + return b, nil +} +func appendFixed32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + return b, nil +} +func appendFixed32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toUint32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, *p) + return b, nil +} +func appendFixed32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + } + return b, nil +} +func appendFixed32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(4*len(s))) + for _, v := range s { + b = appendFixed32(b, v) + } + return b, nil +} +func appendFixedS32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + b = appendVarint(b, wiretag) + b = appendFixed32(b, uint32(v)) + return b, nil +} +func appendFixedS32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, uint32(v)) + return b, nil +} +func appendFixedS32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := ptr.getInt32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, uint32(*p)) + return b, nil +} +func appendFixedS32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed32(b, uint32(v)) + } + return b, nil +} +func appendFixedS32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(4*len(s))) + for _, v := range s { + b = appendFixed32(b, uint32(v)) + } + return b, nil +} +func appendFloat32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := math.Float32bits(*ptr.toFloat32()) + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + return b, nil +} +func appendFloat32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := math.Float32bits(*ptr.toFloat32()) + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, v) + return b, nil +} +func appendFloat32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toFloat32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed32(b, math.Float32bits(*p)) + return b, nil +} +func appendFloat32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toFloat32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed32(b, math.Float32bits(v)) + } + return b, nil +} +func appendFloat32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toFloat32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(4*len(s))) + for _, v := range s { + b = appendFixed32(b, math.Float32bits(v)) + } + return b, nil +} +func appendFixed64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint64() + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + return b, nil +} +func appendFixed64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + return b, nil +} +func appendFixed64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toUint64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, *p) + return b, nil +} +func appendFixed64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + } + return b, nil +} +func appendFixed64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(8*len(s))) + for _, v := range s { + b = appendFixed64(b, v) + } + return b, nil +} +func appendFixedS64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + b = appendVarint(b, wiretag) + b = appendFixed64(b, uint64(v)) + return b, nil +} +func appendFixedS64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, uint64(v)) + return b, nil +} +func appendFixedS64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toInt64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, uint64(*p)) + return b, nil +} +func appendFixedS64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed64(b, uint64(v)) + } + return b, nil +} +func appendFixedS64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(8*len(s))) + for _, v := range s { + b = appendFixed64(b, uint64(v)) + } + return b, nil +} +func appendFloat64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := math.Float64bits(*ptr.toFloat64()) + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + return b, nil +} +func appendFloat64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := math.Float64bits(*ptr.toFloat64()) + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, v) + return b, nil +} +func appendFloat64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toFloat64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendFixed64(b, math.Float64bits(*p)) + return b, nil +} +func appendFloat64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toFloat64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendFixed64(b, math.Float64bits(v)) + } + return b, nil +} +func appendFloat64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toFloat64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(8*len(s))) + for _, v := range s { + b = appendFixed64(b, math.Float64bits(v)) + } + return b, nil +} +func appendVarint32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint32() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarint32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarint32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toUint32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(*p)) + return b, nil +} +func appendVarint32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarint32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarintS32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarintS32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarintS32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := ptr.getInt32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(*p)) + return b, nil +} +func appendVarintS32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarintS32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarint64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint64() + b = appendVarint(b, wiretag) + b = appendVarint(b, v) + return b, nil +} +func appendVarint64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toUint64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, v) + return b, nil +} +func appendVarint64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toUint64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, *p) + return b, nil +} +func appendVarint64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, v) + } + return b, nil +} +func appendVarint64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toUint64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(v) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, v) + } + return b, nil +} +func appendVarintS64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarintS64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + return b, nil +} +func appendVarintS64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toInt64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(*p)) + return b, nil +} +func appendVarintS64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendVarintS64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v)) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64(v)) + } + return b, nil +} +func appendZigzag32Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + return b, nil +} +func appendZigzag32ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt32() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + return b, nil +} +func appendZigzag32Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := ptr.getInt32Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + v := *p + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + return b, nil +} +func appendZigzag32Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + } + return b, nil +} +func appendZigzag32PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := ptr.getInt32Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64((uint32(v) << 1) ^ uint32((int32(v) >> 31)))) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64((uint32(v)<<1)^uint32((int32(v)>>31)))) + } + return b, nil +} +func appendZigzag64Value(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + return b, nil +} +func appendZigzag64ValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toInt64() + if v == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + return b, nil +} +func appendZigzag64Ptr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toInt64Ptr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + v := *p + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + return b, nil +} +func appendZigzag64Slice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + } + return b, nil +} +func appendZigzag64PackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toInt64Slice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + // compute size + n := 0 + for _, v := range s { + n += SizeVarint(uint64(v<<1) ^ uint64((int64(v) >> 63))) + } + b = appendVarint(b, uint64(n)) + for _, v := range s { + b = appendVarint(b, uint64(v<<1)^uint64((int64(v)>>63))) + } + return b, nil +} +func appendBoolValue(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBool() + b = appendVarint(b, wiretag) + if v { + b = append(b, 1) + } else { + b = append(b, 0) + } + return b, nil +} +func appendBoolValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBool() + if !v { + return b, nil + } + b = appendVarint(b, wiretag) + b = append(b, 1) + return b, nil +} + +func appendBoolPtr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toBoolPtr() + if p == nil { + return b, nil + } + b = appendVarint(b, wiretag) + if *p { + b = append(b, 1) + } else { + b = append(b, 0) + } + return b, nil +} +func appendBoolSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toBoolSlice() + for _, v := range s { + b = appendVarint(b, wiretag) + if v { + b = append(b, 1) + } else { + b = append(b, 0) + } + } + return b, nil +} +func appendBoolPackedSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toBoolSlice() + if len(s) == 0 { + return b, nil + } + b = appendVarint(b, wiretag&^7|WireBytes) + b = appendVarint(b, uint64(len(s))) + for _, v := range s { + if v { + b = append(b, 1) + } else { + b = append(b, 0) + } + } + return b, nil +} +func appendStringValue(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toString() + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendStringValueNoZero(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toString() + if v == "" { + return b, nil + } + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendStringPtr(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + p := *ptr.toStringPtr() + if p == nil { + return b, nil + } + v := *p + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendStringSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toStringSlice() + for _, v := range s { + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + } + return b, nil +} +func appendBytes(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBytes() + if v == nil { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendBytes3(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBytes() + if len(v) == 0 { + return b, nil + } + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendBytesOneof(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + v := *ptr.toBytes() + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + return b, nil +} +func appendBytesSlice(b []byte, ptr pointer, wiretag uint64, _ bool) ([]byte, error) { + s := *ptr.toBytesSlice() + for _, v := range s { + b = appendVarint(b, wiretag) + b = appendVarint(b, uint64(len(v))) + b = append(b, v...) + } + return b, nil +} + +// makeGroupMarshaler returns the sizer and marshaler for a group. +// u is the marshal info of the underlying message. +func makeGroupMarshaler(u *marshalInfo) (sizer, marshaler) { + return func(ptr pointer, tagsize int) int { + p := ptr.getPointer() + if p.isNil() { + return 0 + } + return u.size(p) + 2*tagsize + }, + func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) { + p := ptr.getPointer() + if p.isNil() { + return b, nil + } + var err error + b = appendVarint(b, wiretag) // start group + b, err = u.marshal(b, p, deterministic) + b = appendVarint(b, wiretag+(WireEndGroup-WireStartGroup)) // end group + return b, err + } +} + +// makeGroupSliceMarshaler returns the sizer and marshaler for a group slice. +// u is the marshal info of the underlying message. +func makeGroupSliceMarshaler(u *marshalInfo) (sizer, marshaler) { + return func(ptr pointer, tagsize int) int { + s := ptr.getPointerSlice() + n := 0 + for _, v := range s { + if v.isNil() { + continue + } + n += u.size(v) + 2*tagsize + } + return n + }, + func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) { + s := ptr.getPointerSlice() + var err, errreq error + for _, v := range s { + if v.isNil() { + return b, errRepeatedHasNil + } + b = appendVarint(b, wiretag) // start group + b, err = u.marshal(b, v, deterministic) + b = appendVarint(b, wiretag+(WireEndGroup-WireStartGroup)) // end group + if err != nil { + if _, ok := err.(*RequiredNotSetError); ok { + // Required field in submessage is not set. + // We record the error but keep going, to give a complete marshaling. + if errreq == nil { + errreq = err + } + continue + } + if err == ErrNil { + err = errRepeatedHasNil + } + return b, err + } + } + return b, errreq + } +} + +// makeMessageMarshaler returns the sizer and marshaler for a message field. +// u is the marshal info of the message. +func makeMessageMarshaler(u *marshalInfo) (sizer, marshaler) { + return func(ptr pointer, tagsize int) int { + p := ptr.getPointer() + if p.isNil() { + return 0 + } + siz := u.size(p) + return siz + SizeVarint(uint64(siz)) + tagsize + }, + func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) { + p := ptr.getPointer() + if p.isNil() { + return b, nil + } + b = appendVarint(b, wiretag) + siz := u.cachedsize(p) + b = appendVarint(b, uint64(siz)) + return u.marshal(b, p, deterministic) + } +} + +// makeMessageSliceMarshaler returns the sizer and marshaler for a message slice. +// u is the marshal info of the message. +func makeMessageSliceMarshaler(u *marshalInfo) (sizer, marshaler) { + return func(ptr pointer, tagsize int) int { + s := ptr.getPointerSlice() + n := 0 + for _, v := range s { + if v.isNil() { + continue + } + siz := u.size(v) + n += siz + SizeVarint(uint64(siz)) + tagsize + } + return n + }, + func(b []byte, ptr pointer, wiretag uint64, deterministic bool) ([]byte, error) { + s := ptr.getPointerSlice() + var err, errreq error + for _, v := range s { + if v.isNil() { + return b, errRepeatedHasNil + } + b = appendVarint(b, wiretag) + siz := u.cachedsize(v) + b = appendVarint(b, uint64(siz)) + b, err = u.marshal(b, v, deterministic) + + if err != nil { + if _, ok := err.(*RequiredNotSetError); ok { + // Required field in submessage is not set. + // We record the error but keep going, to give a complete marshaling. + if errreq == nil { + errreq = err + } + continue + } + if err == ErrNil { + err = errRepeatedHasNil + } + return b, err + } + } + return b, errreq + } +} + +// makeMapMarshaler returns the sizer and marshaler for a map field. +// f is the pointer to the reflect data structure of the field. +func makeMapMarshaler(f *reflect.StructField) (sizer, marshaler) { + // figure out key and value type + t := f.Type + keyType := t.Key() + valType := t.Elem() + keyTags := strings.Split(f.Tag.Get("protobuf_key"), ",") + valTags := strings.Split(f.Tag.Get("protobuf_val"), ",") + keySizer, keyMarshaler := typeMarshaler(keyType, keyTags, false, false) // don't omit zero value in map + valSizer, valMarshaler := typeMarshaler(valType, valTags, false, false) // don't omit zero value in map + keyWireTag := 1<<3 | wiretype(keyTags[0]) + valWireTag := 2<<3 | wiretype(valTags[0]) + + // We create an interface to get the addresses of the map key and value. + // If value is pointer-typed, the interface is a direct interface, the + // idata itself is the value. Otherwise, the idata is the pointer to the + // value. + // Key cannot be pointer-typed. + valIsPtr := valType.Kind() == reflect.Ptr + return func(ptr pointer, tagsize int) int { + m := ptr.asPointerTo(t).Elem() // the map + n := 0 + for _, k := range m.MapKeys() { + ki := k.Interface() + vi := m.MapIndex(k).Interface() + kaddr := toAddrPointer(&ki, false) // pointer to key + vaddr := toAddrPointer(&vi, valIsPtr) // pointer to value + siz := keySizer(kaddr, 1) + valSizer(vaddr, 1) // tag of key = 1 (size=1), tag of val = 2 (size=1) + n += siz + SizeVarint(uint64(siz)) + tagsize + } + return n + }, + func(b []byte, ptr pointer, tag uint64, deterministic bool) ([]byte, error) { + m := ptr.asPointerTo(t).Elem() // the map + var err error + keys := m.MapKeys() + if len(keys) > 1 && deterministic { + sort.Sort(mapKeys(keys)) + } + for _, k := range keys { + ki := k.Interface() + vi := m.MapIndex(k).Interface() + kaddr := toAddrPointer(&ki, false) // pointer to key + vaddr := toAddrPointer(&vi, valIsPtr) // pointer to value + b = appendVarint(b, tag) + siz := keySizer(kaddr, 1) + valSizer(vaddr, 1) // tag of key = 1 (size=1), tag of val = 2 (size=1) + b = appendVarint(b, uint64(siz)) + b, err = keyMarshaler(b, kaddr, keyWireTag, deterministic) + if err != nil { + return b, err + } + b, err = valMarshaler(b, vaddr, valWireTag, deterministic) + if err != nil && err != ErrNil { // allow nil value in map + return b, err + } + } + return b, nil + } +} + +// makeOneOfMarshaler returns the sizer and marshaler for a oneof field. +// fi is the marshal info of the field. +// f is the pointer to the reflect data structure of the field. +func makeOneOfMarshaler(fi *marshalFieldInfo, f *reflect.StructField) (sizer, marshaler) { + // Oneof field is an interface. We need to get the actual data type on the fly. + t := f.Type + return func(ptr pointer, _ int) int { + p := ptr.getInterfacePointer() + if p.isNil() { + return 0 + } + v := ptr.asPointerTo(t).Elem().Elem().Elem() // *interface -> interface -> *struct -> struct + telem := v.Type() + e := fi.oneofElems[telem] + return e.sizer(p, e.tagsize) + }, + func(b []byte, ptr pointer, _ uint64, deterministic bool) ([]byte, error) { + p := ptr.getInterfacePointer() + if p.isNil() { + return b, nil + } + v := ptr.asPointerTo(t).Elem().Elem().Elem() // *interface -> interface -> *struct -> struct + telem := v.Type() + if telem.Field(0).Type.Kind() == reflect.Ptr && p.getPointer().isNil() { + return b, errOneofHasNil + } + e := fi.oneofElems[telem] + return e.marshaler(b, p, e.wiretag, deterministic) + } +} + +// sizeExtensions computes the size of encoded data for a XXX_InternalExtensions field. +func (u *marshalInfo) sizeExtensions(ext *XXX_InternalExtensions) int { + m, mu := ext.extensionsRead() + if m == nil { + return 0 + } + mu.Lock() + + n := 0 + for _, e := range m { + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + n += len(e.enc) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + n += ei.sizer(p, ei.tagsize) + } + mu.Unlock() + return n +} + +// appendExtensions marshals a XXX_InternalExtensions field to the end of byte slice b. +func (u *marshalInfo) appendExtensions(b []byte, ext *XXX_InternalExtensions, deterministic bool) ([]byte, error) { + m, mu := ext.extensionsRead() + if m == nil { + return b, nil + } + mu.Lock() + defer mu.Unlock() + + var err error + + // Fast-path for common cases: zero or one extensions. + // Don't bother sorting the keys. + if len(m) <= 1 { + for _, e := range m { + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + b = append(b, e.enc...) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, ei.wiretag, deterministic) + if err != nil { + return b, err + } + } + return b, nil + } + + // Sort the keys to provide a deterministic encoding. + // Not sure this is required, but the old code does it. + keys := make([]int, 0, len(m)) + for k := range m { + keys = append(keys, int(k)) + } + sort.Ints(keys) + + for _, k := range keys { + e := m[int32(k)] + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + b = append(b, e.enc...) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, ei.wiretag, deterministic) + if err != nil { + return b, err + } + } + return b, nil +} + +// message set format is: +// message MessageSet { +// repeated group Item = 1 { +// required int32 type_id = 2; +// required string message = 3; +// }; +// } + +// sizeMessageSet computes the size of encoded data for a XXX_InternalExtensions field +// in message set format (above). +func (u *marshalInfo) sizeMessageSet(ext *XXX_InternalExtensions) int { + m, mu := ext.extensionsRead() + if m == nil { + return 0 + } + mu.Lock() + + n := 0 + for id, e := range m { + n += 2 // start group, end group. tag = 1 (size=1) + n += SizeVarint(uint64(id)) + 1 // type_id, tag = 2 (size=1) + + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + msgWithLen := skipVarint(e.enc) // skip old tag, but leave the length varint + siz := len(msgWithLen) + n += siz + 1 // message, tag = 3 (size=1) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + n += ei.sizer(p, 1) // message, tag = 3 (size=1) + } + mu.Unlock() + return n +} + +// appendMessageSet marshals a XXX_InternalExtensions field in message set format (above) +// to the end of byte slice b. +func (u *marshalInfo) appendMessageSet(b []byte, ext *XXX_InternalExtensions, deterministic bool) ([]byte, error) { + m, mu := ext.extensionsRead() + if m == nil { + return b, nil + } + mu.Lock() + defer mu.Unlock() + + var err error + + // Fast-path for common cases: zero or one extensions. + // Don't bother sorting the keys. + if len(m) <= 1 { + for id, e := range m { + b = append(b, 1<<3|WireStartGroup) + b = append(b, 2<<3|WireVarint) + b = appendVarint(b, uint64(id)) + + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + msgWithLen := skipVarint(e.enc) // skip old tag, but leave the length varint + b = append(b, 3<<3|WireBytes) + b = append(b, msgWithLen...) + b = append(b, 1<<3|WireEndGroup) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, 3<<3|WireBytes, deterministic) + if err != nil { + return b, err + } + b = append(b, 1<<3|WireEndGroup) + } + return b, nil + } + + // Sort the keys to provide a deterministic encoding. + keys := make([]int, 0, len(m)) + for k := range m { + keys = append(keys, int(k)) + } + sort.Ints(keys) + + for _, id := range keys { + e := m[int32(id)] + b = append(b, 1<<3|WireStartGroup) + b = append(b, 2<<3|WireVarint) + b = appendVarint(b, uint64(id)) + + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + msgWithLen := skipVarint(e.enc) // skip old tag, but leave the length varint + b = append(b, 3<<3|WireBytes) + b = append(b, msgWithLen...) + b = append(b, 1<<3|WireEndGroup) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, 3<<3|WireBytes, deterministic) + b = append(b, 1<<3|WireEndGroup) + if err != nil { + return b, err + } + } + return b, nil +} + +// sizeV1Extensions computes the size of encoded data for a V1-API extension field. +func (u *marshalInfo) sizeV1Extensions(m map[int32]Extension) int { + if m == nil { + return 0 + } + + n := 0 + for _, e := range m { + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + n += len(e.enc) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + n += ei.sizer(p, ei.tagsize) + } + return n +} + +// appendV1Extensions marshals a V1-API extension field to the end of byte slice b. +func (u *marshalInfo) appendV1Extensions(b []byte, m map[int32]Extension, deterministic bool) ([]byte, error) { + if m == nil { + return b, nil + } + + // Sort the keys to provide a deterministic encoding. + keys := make([]int, 0, len(m)) + for k := range m { + keys = append(keys, int(k)) + } + sort.Ints(keys) + + var err error + for _, k := range keys { + e := m[int32(k)] + if e.value == nil || e.desc == nil { + // Extension is only in its encoded form. + b = append(b, e.enc...) + continue + } + + // We don't skip extensions that have an encoded form set, + // because the extension value may have been mutated after + // the last time this function was called. + + ei := u.getExtElemInfo(e.desc) + v := e.value + p := toAddrPointer(&v, ei.isptr) + b, err = ei.marshaler(b, p, ei.wiretag, deterministic) + if err != nil { + return b, err + } + } + return b, nil +} + +// newMarshaler is the interface representing objects that can marshal themselves. +// +// This exists to support protoc-gen-go generated messages. +// The proto package will stop type-asserting to this interface in the future. +// +// DO NOT DEPEND ON THIS. +type newMarshaler interface { + XXX_Size() int + XXX_Marshal(b []byte, deterministic bool) ([]byte, error) +} + +// Size returns the encoded size of a protocol buffer message. +// This is the main entry point. +func Size(pb Message) int { + if m, ok := pb.(newMarshaler); ok { + return m.XXX_Size() + } + if m, ok := pb.(Marshaler); ok { + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + b, _ := m.Marshal() + return len(b) + } + // in case somehow we didn't generate the wrapper + if pb == nil { + return 0 + } + var info InternalMessageInfo + return info.Size(pb) +} + +// Marshal takes a protocol buffer message +// and encodes it into the wire format, returning the data. +// This is the main entry point. +func Marshal(pb Message) ([]byte, error) { + if m, ok := pb.(newMarshaler); ok { + siz := m.XXX_Size() + b := make([]byte, 0, siz) + return m.XXX_Marshal(b, false) + } + if m, ok := pb.(Marshaler); ok { + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + return m.Marshal() + } + // in case somehow we didn't generate the wrapper + if pb == nil { + return nil, ErrNil + } + var info InternalMessageInfo + siz := info.Size(pb) + b := make([]byte, 0, siz) + return info.Marshal(b, pb, false) +} + +// Marshal takes a protocol buffer message +// and encodes it into the wire format, writing the result to the +// Buffer. +// This is an alternative entry point. It is not necessary to use +// a Buffer for most applications. +func (p *Buffer) Marshal(pb Message) error { + var err error + if m, ok := pb.(newMarshaler); ok { + siz := m.XXX_Size() + p.grow(siz) // make sure buf has enough capacity + p.buf, err = m.XXX_Marshal(p.buf, p.deterministic) + return err + } + if m, ok := pb.(Marshaler); ok { + // If the message can marshal itself, let it do it, for compatibility. + // NOTE: This is not efficient. + b, err := m.Marshal() + p.buf = append(p.buf, b...) + return err + } + // in case somehow we didn't generate the wrapper + if pb == nil { + return ErrNil + } + var info InternalMessageInfo + siz := info.Size(pb) + p.grow(siz) // make sure buf has enough capacity + p.buf, err = info.Marshal(p.buf, pb, p.deterministic) + return err +} + +// grow grows the buffer's capacity, if necessary, to guarantee space for +// another n bytes. After grow(n), at least n bytes can be written to the +// buffer without another allocation. +func (p *Buffer) grow(n int) { + need := len(p.buf) + n + if need <= cap(p.buf) { + return + } + newCap := len(p.buf) * 2 + if newCap < need { + newCap = need + } + p.buf = append(make([]byte, 0, newCap), p.buf...) +} diff --git a/vendor/github.com/golang/protobuf/proto/table_merge.go b/vendor/github.com/golang/protobuf/proto/table_merge.go new file mode 100644 index 00000000..5525def6 --- /dev/null +++ b/vendor/github.com/golang/protobuf/proto/table_merge.go @@ -0,0 +1,654 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" +) + +// Merge merges the src message into dst. +// This assumes that dst and src of the same type and are non-nil. +func (a *InternalMessageInfo) Merge(dst, src Message) { + mi := atomicLoadMergeInfo(&a.merge) + if mi == nil { + mi = getMergeInfo(reflect.TypeOf(dst).Elem()) + atomicStoreMergeInfo(&a.merge, mi) + } + mi.merge(toPointer(&dst), toPointer(&src)) +} + +type mergeInfo struct { + typ reflect.Type + + initialized int32 // 0: only typ is valid, 1: everything is valid + lock sync.Mutex + + fields []mergeFieldInfo + unrecognized field // Offset of XXX_unrecognized +} + +type mergeFieldInfo struct { + field field // Offset of field, guaranteed to be valid + + // isPointer reports whether the value in the field is a pointer. + // This is true for the following situations: + // * Pointer to struct + // * Pointer to basic type (proto2 only) + // * Slice (first value in slice header is a pointer) + // * String (first value in string header is a pointer) + isPointer bool + + // basicWidth reports the width of the field assuming that it is directly + // embedded in the struct (as is the case for basic types in proto3). + // The possible values are: + // 0: invalid + // 1: bool + // 4: int32, uint32, float32 + // 8: int64, uint64, float64 + basicWidth int + + // Where dst and src are pointers to the types being merged. + merge func(dst, src pointer) +} + +var ( + mergeInfoMap = map[reflect.Type]*mergeInfo{} + mergeInfoLock sync.Mutex +) + +func getMergeInfo(t reflect.Type) *mergeInfo { + mergeInfoLock.Lock() + defer mergeInfoLock.Unlock() + mi := mergeInfoMap[t] + if mi == nil { + mi = &mergeInfo{typ: t} + mergeInfoMap[t] = mi + } + return mi +} + +// merge merges src into dst assuming they are both of type *mi.typ. +func (mi *mergeInfo) merge(dst, src pointer) { + if dst.isNil() { + panic("proto: nil destination") + } + if src.isNil() { + return // Nothing to do. + } + + if atomic.LoadInt32(&mi.initialized) == 0 { + mi.computeMergeInfo() + } + + for _, fi := range mi.fields { + sfp := src.offset(fi.field) + + // As an optimization, we can avoid the merge function call cost + // if we know for sure that the source will have no effect + // by checking if it is the zero value. + if unsafeAllowed { + if fi.isPointer && sfp.getPointer().isNil() { // Could be slice or string + continue + } + if fi.basicWidth > 0 { + switch { + case fi.basicWidth == 1 && !*sfp.toBool(): + continue + case fi.basicWidth == 4 && *sfp.toUint32() == 0: + continue + case fi.basicWidth == 8 && *sfp.toUint64() == 0: + continue + } + } + } + + dfp := dst.offset(fi.field) + fi.merge(dfp, sfp) + } + + // TODO: Make this faster? + out := dst.asPointerTo(mi.typ).Elem() + in := src.asPointerTo(mi.typ).Elem() + if emIn, err := extendable(in.Addr().Interface()); err == nil { + emOut, _ := extendable(out.Addr().Interface()) + mIn, muIn := emIn.extensionsRead() + if mIn != nil { + mOut := emOut.extensionsWrite() + muIn.Lock() + mergeExtension(mOut, mIn) + muIn.Unlock() + } + } + + if mi.unrecognized.IsValid() { + if b := *src.offset(mi.unrecognized).toBytes(); len(b) > 0 { + *dst.offset(mi.unrecognized).toBytes() = append([]byte(nil), b...) + } + } +} + +func (mi *mergeInfo) computeMergeInfo() { + mi.lock.Lock() + defer mi.lock.Unlock() + if mi.initialized != 0 { + return + } + t := mi.typ + n := t.NumField() + + props := GetProperties(t) + for i := 0; i < n; i++ { + f := t.Field(i) + if strings.HasPrefix(f.Name, "XXX_") { + continue + } + + mfi := mergeFieldInfo{field: toField(&f)} + tf := f.Type + + // As an optimization, we can avoid the merge function call cost + // if we know for sure that the source will have no effect + // by checking if it is the zero value. + if unsafeAllowed { + switch tf.Kind() { + case reflect.Ptr, reflect.Slice, reflect.String: + // As a special case, we assume slices and strings are pointers + // since we know that the first field in the SliceSlice or + // StringHeader is a data pointer. + mfi.isPointer = true + case reflect.Bool: + mfi.basicWidth = 1 + case reflect.Int32, reflect.Uint32, reflect.Float32: + mfi.basicWidth = 4 + case reflect.Int64, reflect.Uint64, reflect.Float64: + mfi.basicWidth = 8 + } + } + + // Unwrap tf to get at its most basic type. + var isPointer, isSlice bool + if tf.Kind() == reflect.Slice && tf.Elem().Kind() != reflect.Uint8 { + isSlice = true + tf = tf.Elem() + } + if tf.Kind() == reflect.Ptr { + isPointer = true + tf = tf.Elem() + } + if isPointer && isSlice && tf.Kind() != reflect.Struct { + panic("both pointer and slice for basic type in " + tf.Name()) + } + + switch tf.Kind() { + case reflect.Int32: + switch { + case isSlice: // E.g., []int32 + mfi.merge = func(dst, src pointer) { + // NOTE: toInt32Slice is not defined (see pointer_reflect.go). + /* + sfsp := src.toInt32Slice() + if *sfsp != nil { + dfsp := dst.toInt32Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []int64{} + } + } + */ + sfs := src.getInt32Slice() + if sfs != nil { + dfs := dst.getInt32Slice() + dfs = append(dfs, sfs...) + if dfs == nil { + dfs = []int32{} + } + dst.setInt32Slice(dfs) + } + } + case isPointer: // E.g., *int32 + mfi.merge = func(dst, src pointer) { + // NOTE: toInt32Ptr is not defined (see pointer_reflect.go). + /* + sfpp := src.toInt32Ptr() + if *sfpp != nil { + dfpp := dst.toInt32Ptr() + if *dfpp == nil { + *dfpp = Int32(**sfpp) + } else { + **dfpp = **sfpp + } + } + */ + sfp := src.getInt32Ptr() + if sfp != nil { + dfp := dst.getInt32Ptr() + if dfp == nil { + dst.setInt32Ptr(*sfp) + } else { + *dfp = *sfp + } + } + } + default: // E.g., int32 + mfi.merge = func(dst, src pointer) { + if v := *src.toInt32(); v != 0 { + *dst.toInt32() = v + } + } + } + case reflect.Int64: + switch { + case isSlice: // E.g., []int64 + mfi.merge = func(dst, src pointer) { + sfsp := src.toInt64Slice() + if *sfsp != nil { + dfsp := dst.toInt64Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []int64{} + } + } + } + case isPointer: // E.g., *int64 + mfi.merge = func(dst, src pointer) { + sfpp := src.toInt64Ptr() + if *sfpp != nil { + dfpp := dst.toInt64Ptr() + if *dfpp == nil { + *dfpp = Int64(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., int64 + mfi.merge = func(dst, src pointer) { + if v := *src.toInt64(); v != 0 { + *dst.toInt64() = v + } + } + } + case reflect.Uint32: + switch { + case isSlice: // E.g., []uint32 + mfi.merge = func(dst, src pointer) { + sfsp := src.toUint32Slice() + if *sfsp != nil { + dfsp := dst.toUint32Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []uint32{} + } + } + } + case isPointer: // E.g., *uint32 + mfi.merge = func(dst, src pointer) { + sfpp := src.toUint32Ptr() + if *sfpp != nil { + dfpp := dst.toUint32Ptr() + if *dfpp == nil { + *dfpp = Uint32(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., uint32 + mfi.merge = func(dst, src pointer) { + if v := *src.toUint32(); v != 0 { + *dst.toUint32() = v + } + } + } + case reflect.Uint64: + switch { + case isSlice: // E.g., []uint64 + mfi.merge = func(dst, src pointer) { + sfsp := src.toUint64Slice() + if *sfsp != nil { + dfsp := dst.toUint64Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []uint64{} + } + } + } + case isPointer: // E.g., *uint64 + mfi.merge = func(dst, src pointer) { + sfpp := src.toUint64Ptr() + if *sfpp != nil { + dfpp := dst.toUint64Ptr() + if *dfpp == nil { + *dfpp = Uint64(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., uint64 + mfi.merge = func(dst, src pointer) { + if v := *src.toUint64(); v != 0 { + *dst.toUint64() = v + } + } + } + case reflect.Float32: + switch { + case isSlice: // E.g., []float32 + mfi.merge = func(dst, src pointer) { + sfsp := src.toFloat32Slice() + if *sfsp != nil { + dfsp := dst.toFloat32Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []float32{} + } + } + } + case isPointer: // E.g., *float32 + mfi.merge = func(dst, src pointer) { + sfpp := src.toFloat32Ptr() + if *sfpp != nil { + dfpp := dst.toFloat32Ptr() + if *dfpp == nil { + *dfpp = Float32(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., float32 + mfi.merge = func(dst, src pointer) { + if v := *src.toFloat32(); v != 0 { + *dst.toFloat32() = v + } + } + } + case reflect.Float64: + switch { + case isSlice: // E.g., []float64 + mfi.merge = func(dst, src pointer) { + sfsp := src.toFloat64Slice() + if *sfsp != nil { + dfsp := dst.toFloat64Slice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []float64{} + } + } + } + case isPointer: // E.g., *float64 + mfi.merge = func(dst, src pointer) { + sfpp := src.toFloat64Ptr() + if *sfpp != nil { + dfpp := dst.toFloat64Ptr() + if *dfpp == nil { + *dfpp = Float64(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., float64 + mfi.merge = func(dst, src pointer) { + if v := *src.toFloat64(); v != 0 { + *dst.toFloat64() = v + } + } + } + case reflect.Bool: + switch { + case isSlice: // E.g., []bool + mfi.merge = func(dst, src pointer) { + sfsp := src.toBoolSlice() + if *sfsp != nil { + dfsp := dst.toBoolSlice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []bool{} + } + } + } + case isPointer: // E.g., *bool + mfi.merge = func(dst, src pointer) { + sfpp := src.toBoolPtr() + if *sfpp != nil { + dfpp := dst.toBoolPtr() + if *dfpp == nil { + *dfpp = Bool(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., bool + mfi.merge = func(dst, src pointer) { + if v := *src.toBool(); v { + *dst.toBool() = v + } + } + } + case reflect.String: + switch { + case isSlice: // E.g., []string + mfi.merge = func(dst, src pointer) { + sfsp := src.toStringSlice() + if *sfsp != nil { + dfsp := dst.toStringSlice() + *dfsp = append(*dfsp, *sfsp...) + if *dfsp == nil { + *dfsp = []string{} + } + } + } + case isPointer: // E.g., *string + mfi.merge = func(dst, src pointer) { + sfpp := src.toStringPtr() + if *sfpp != nil { + dfpp := dst.toStringPtr() + if *dfpp == nil { + *dfpp = String(**sfpp) + } else { + **dfpp = **sfpp + } + } + } + default: // E.g., string + mfi.merge = func(dst, src pointer) { + if v := *src.toString(); v != "" { + *dst.toString() = v + } + } + } + case reflect.Slice: + isProto3 := props.Prop[i].proto3 + switch { + case isPointer: + panic("bad pointer in byte slice case in " + tf.Name()) + case tf.Elem().Kind() != reflect.Uint8: + panic("bad element kind in byte slice case in " + tf.Name()) + case isSlice: // E.g., [][]byte + mfi.merge = func(dst, src pointer) { + sbsp := src.toBytesSlice() + if *sbsp != nil { + dbsp := dst.toBytesSlice() + for _, sb := range *sbsp { + if sb == nil { + *dbsp = append(*dbsp, nil) + } else { + *dbsp = append(*dbsp, append([]byte{}, sb...)) + } + } + if *dbsp == nil { + *dbsp = [][]byte{} + } + } + } + default: // E.g., []byte + mfi.merge = func(dst, src pointer) { + sbp := src.toBytes() + if *sbp != nil { + dbp := dst.toBytes() + if !isProto3 || len(*sbp) > 0 { + *dbp = append([]byte{}, *sbp...) + } + } + } + } + case reflect.Struct: + switch { + case !isPointer: + panic(fmt.Sprintf("message field %s without pointer", tf)) + case isSlice: // E.g., []*pb.T + mi := getMergeInfo(tf) + mfi.merge = func(dst, src pointer) { + sps := src.getPointerSlice() + if sps != nil { + dps := dst.getPointerSlice() + for _, sp := range sps { + var dp pointer + if !sp.isNil() { + dp = valToPointer(reflect.New(tf)) + mi.merge(dp, sp) + } + dps = append(dps, dp) + } + if dps == nil { + dps = []pointer{} + } + dst.setPointerSlice(dps) + } + } + default: // E.g., *pb.T + mi := getMergeInfo(tf) + mfi.merge = func(dst, src pointer) { + sp := src.getPointer() + if !sp.isNil() { + dp := dst.getPointer() + if dp.isNil() { + dp = valToPointer(reflect.New(tf)) + dst.setPointer(dp) + } + mi.merge(dp, sp) + } + } + } + case reflect.Map: + switch { + case isPointer || isSlice: + panic("bad pointer or slice in map case in " + tf.Name()) + default: // E.g., map[K]V + mfi.merge = func(dst, src pointer) { + sm := src.asPointerTo(tf).Elem() + if sm.Len() == 0 { + return + } + dm := dst.asPointerTo(tf).Elem() + if dm.IsNil() { + dm.Set(reflect.MakeMap(tf)) + } + + switch tf.Elem().Kind() { + case reflect.Ptr: // Proto struct (e.g., *T) + for _, key := range sm.MapKeys() { + val := sm.MapIndex(key) + val = reflect.ValueOf(Clone(val.Interface().(Message))) + dm.SetMapIndex(key, val) + } + case reflect.Slice: // E.g. Bytes type (e.g., []byte) + for _, key := range sm.MapKeys() { + val := sm.MapIndex(key) + val = reflect.ValueOf(append([]byte{}, val.Bytes()...)) + dm.SetMapIndex(key, val) + } + default: // Basic type (e.g., string) + for _, key := range sm.MapKeys() { + val := sm.MapIndex(key) + dm.SetMapIndex(key, val) + } + } + } + } + case reflect.Interface: + // Must be oneof field. + switch { + case isPointer || isSlice: + panic("bad pointer or slice in interface case in " + tf.Name()) + default: // E.g., interface{} + // TODO: Make this faster? + mfi.merge = func(dst, src pointer) { + su := src.asPointerTo(tf).Elem() + if !su.IsNil() { + du := dst.asPointerTo(tf).Elem() + typ := su.Elem().Type() + if du.IsNil() || du.Elem().Type() != typ { + du.Set(reflect.New(typ.Elem())) // Initialize interface if empty + } + sv := su.Elem().Elem().Field(0) + if sv.Kind() == reflect.Ptr && sv.IsNil() { + return + } + dv := du.Elem().Elem().Field(0) + if dv.Kind() == reflect.Ptr && dv.IsNil() { + dv.Set(reflect.New(sv.Type().Elem())) // Initialize proto message if empty + } + switch sv.Type().Kind() { + case reflect.Ptr: // Proto struct (e.g., *T) + Merge(dv.Interface().(Message), sv.Interface().(Message)) + case reflect.Slice: // E.g. Bytes type (e.g., []byte) + dv.Set(reflect.ValueOf(append([]byte{}, sv.Bytes()...))) + default: // Basic type (e.g., string) + dv.Set(sv) + } + } + } + } + default: + panic(fmt.Sprintf("merger not found for type:%s", tf)) + } + mi.fields = append(mi.fields, mfi) + } + + mi.unrecognized = invalidField + if f, ok := t.FieldByName("XXX_unrecognized"); ok { + if f.Type != reflect.TypeOf([]byte{}) { + panic("expected XXX_unrecognized to be of type []byte") + } + mi.unrecognized = toField(&f) + } + + atomic.StoreInt32(&mi.initialized, 1) +} diff --git a/vendor/github.com/golang/protobuf/proto/table_unmarshal.go b/vendor/github.com/golang/protobuf/proto/table_unmarshal.go new file mode 100644 index 00000000..55f0340a --- /dev/null +++ b/vendor/github.com/golang/protobuf/proto/table_unmarshal.go @@ -0,0 +1,1967 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2016 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto + +import ( + "errors" + "fmt" + "io" + "math" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + "unicode/utf8" +) + +// Unmarshal is the entry point from the generated .pb.go files. +// This function is not intended to be used by non-generated code. +// This function is not subject to any compatibility guarantee. +// msg contains a pointer to a protocol buffer struct. +// b is the data to be unmarshaled into the protocol buffer. +// a is a pointer to a place to store cached unmarshal information. +func (a *InternalMessageInfo) Unmarshal(msg Message, b []byte) error { + // Load the unmarshal information for this message type. + // The atomic load ensures memory consistency. + u := atomicLoadUnmarshalInfo(&a.unmarshal) + if u == nil { + // Slow path: find unmarshal info for msg, update a with it. + u = getUnmarshalInfo(reflect.TypeOf(msg).Elem()) + atomicStoreUnmarshalInfo(&a.unmarshal, u) + } + // Then do the unmarshaling. + err := u.unmarshal(toPointer(&msg), b) + return err +} + +type unmarshalInfo struct { + typ reflect.Type // type of the protobuf struct + + // 0 = only typ field is initialized + // 1 = completely initialized + initialized int32 + lock sync.Mutex // prevents double initialization + dense []unmarshalFieldInfo // fields indexed by tag # + sparse map[uint64]unmarshalFieldInfo // fields indexed by tag # + reqFields []string // names of required fields + reqMask uint64 // 1< 0 { + // Read tag and wire type. + // Special case 1 and 2 byte varints. + var x uint64 + if b[0] < 128 { + x = uint64(b[0]) + b = b[1:] + } else if len(b) >= 2 && b[1] < 128 { + x = uint64(b[0]&0x7f) + uint64(b[1])<<7 + b = b[2:] + } else { + var n int + x, n = decodeVarint(b) + if n == 0 { + return io.ErrUnexpectedEOF + } + b = b[n:] + } + tag := x >> 3 + wire := int(x) & 7 + + // Dispatch on the tag to one of the unmarshal* functions below. + var f unmarshalFieldInfo + if tag < uint64(len(u.dense)) { + f = u.dense[tag] + } else { + f = u.sparse[tag] + } + if fn := f.unmarshal; fn != nil { + var err error + b, err = fn(b, m.offset(f.field), wire) + if err == nil { + reqMask |= f.reqMask + continue + } + if r, ok := err.(*RequiredNotSetError); ok { + // Remember this error, but keep parsing. We need to produce + // a full parse even if a required field is missing. + rnse = r + reqMask |= f.reqMask + continue + } + if err != errInternalBadWireType { + return err + } + // Fragments with bad wire type are treated as unknown fields. + } + + // Unknown tag. + if !u.unrecognized.IsValid() { + // Don't keep unrecognized data; just skip it. + var err error + b, err = skipField(b, wire) + if err != nil { + return err + } + continue + } + // Keep unrecognized data around. + // maybe in extensions, maybe in the unrecognized field. + z := m.offset(u.unrecognized).toBytes() + var emap map[int32]Extension + var e Extension + for _, r := range u.extensionRanges { + if uint64(r.Start) <= tag && tag <= uint64(r.End) { + if u.extensions.IsValid() { + mp := m.offset(u.extensions).toExtensions() + emap = mp.extensionsWrite() + e = emap[int32(tag)] + z = &e.enc + break + } + if u.oldExtensions.IsValid() { + p := m.offset(u.oldExtensions).toOldExtensions() + emap = *p + if emap == nil { + emap = map[int32]Extension{} + *p = emap + } + e = emap[int32(tag)] + z = &e.enc + break + } + panic("no extensions field available") + } + } + + // Use wire type to skip data. + var err error + b0 := b + b, err = skipField(b, wire) + if err != nil { + return err + } + *z = encodeVarint(*z, tag<<3|uint64(wire)) + *z = append(*z, b0[:len(b0)-len(b)]...) + + if emap != nil { + emap[int32(tag)] = e + } + } + if rnse != nil { + // A required field of a submessage/group is missing. Return that error. + return rnse + } + if reqMask != u.reqMask { + // A required field of this message is missing. + for _, n := range u.reqFields { + if reqMask&1 == 0 { + return &RequiredNotSetError{n} + } + reqMask >>= 1 + } + } + return nil +} + +// computeUnmarshalInfo fills in u with information for use +// in unmarshaling protocol buffers of type u.typ. +func (u *unmarshalInfo) computeUnmarshalInfo() { + u.lock.Lock() + defer u.lock.Unlock() + if u.initialized != 0 { + return + } + t := u.typ + n := t.NumField() + + // Set up the "not found" value for the unrecognized byte buffer. + // This is the default for proto3. + u.unrecognized = invalidField + u.extensions = invalidField + u.oldExtensions = invalidField + + // List of the generated type and offset for each oneof field. + type oneofField struct { + ityp reflect.Type // interface type of oneof field + field field // offset in containing message + } + var oneofFields []oneofField + + for i := 0; i < n; i++ { + f := t.Field(i) + if f.Name == "XXX_unrecognized" { + // The byte slice used to hold unrecognized input is special. + if f.Type != reflect.TypeOf(([]byte)(nil)) { + panic("bad type for XXX_unrecognized field: " + f.Type.Name()) + } + u.unrecognized = toField(&f) + continue + } + if f.Name == "XXX_InternalExtensions" { + // Ditto here. + if f.Type != reflect.TypeOf(XXX_InternalExtensions{}) { + panic("bad type for XXX_InternalExtensions field: " + f.Type.Name()) + } + u.extensions = toField(&f) + if f.Tag.Get("protobuf_messageset") == "1" { + u.isMessageSet = true + } + continue + } + if f.Name == "XXX_extensions" { + // An older form of the extensions field. + if f.Type != reflect.TypeOf((map[int32]Extension)(nil)) { + panic("bad type for XXX_extensions field: " + f.Type.Name()) + } + u.oldExtensions = toField(&f) + continue + } + if f.Name == "XXX_NoUnkeyedLiteral" || f.Name == "XXX_sizecache" { + continue + } + + oneof := f.Tag.Get("protobuf_oneof") + if oneof != "" { + oneofFields = append(oneofFields, oneofField{f.Type, toField(&f)}) + // The rest of oneof processing happens below. + continue + } + + tags := f.Tag.Get("protobuf") + tagArray := strings.Split(tags, ",") + if len(tagArray) < 2 { + panic("protobuf tag not enough fields in " + t.Name() + "." + f.Name + ": " + tags) + } + tag, err := strconv.Atoi(tagArray[1]) + if err != nil { + panic("protobuf tag field not an integer: " + tagArray[1]) + } + + name := "" + for _, tag := range tagArray[3:] { + if strings.HasPrefix(tag, "name=") { + name = tag[5:] + } + } + + // Extract unmarshaling function from the field (its type and tags). + unmarshal := fieldUnmarshaler(&f) + + // Required field? + var reqMask uint64 + if tagArray[2] == "req" { + bit := len(u.reqFields) + u.reqFields = append(u.reqFields, name) + reqMask = uint64(1) << uint(bit) + // TODO: if we have more than 64 required fields, we end up + // not verifying that all required fields are present. + // Fix this, perhaps using a count of required fields? + } + + // Store the info in the correct slot in the message. + u.setTag(tag, toField(&f), unmarshal, reqMask) + } + + // Find any types associated with oneof fields. + // TODO: XXX_OneofFuncs returns more info than we need. Get rid of some of it? + fn := reflect.Zero(reflect.PtrTo(t)).MethodByName("XXX_OneofFuncs") + if fn.IsValid() { + res := fn.Call(nil)[3] // last return value from XXX_OneofFuncs: []interface{} + for i := res.Len() - 1; i >= 0; i-- { + v := res.Index(i) // interface{} + tptr := reflect.ValueOf(v.Interface()).Type() // *Msg_X + typ := tptr.Elem() // Msg_X + + f := typ.Field(0) // oneof implementers have one field + baseUnmarshal := fieldUnmarshaler(&f) + tagstr := strings.Split(f.Tag.Get("protobuf"), ",")[1] + tag, err := strconv.Atoi(tagstr) + if err != nil { + panic("protobuf tag field not an integer: " + tagstr) + } + + // Find the oneof field that this struct implements. + // Might take O(n^2) to process all of the oneofs, but who cares. + for _, of := range oneofFields { + if tptr.Implements(of.ityp) { + // We have found the corresponding interface for this struct. + // That lets us know where this struct should be stored + // when we encounter it during unmarshaling. + unmarshal := makeUnmarshalOneof(typ, of.ityp, baseUnmarshal) + u.setTag(tag, of.field, unmarshal, 0) + } + } + } + } + + // Get extension ranges, if any. + fn = reflect.Zero(reflect.PtrTo(t)).MethodByName("ExtensionRangeArray") + if fn.IsValid() { + if !u.extensions.IsValid() && !u.oldExtensions.IsValid() { + panic("a message with extensions, but no extensions field in " + t.Name()) + } + u.extensionRanges = fn.Call(nil)[0].Interface().([]ExtensionRange) + } + + // Explicitly disallow tag 0. This will ensure we flag an error + // when decoding a buffer of all zeros. Without this code, we + // would decode and skip an all-zero buffer of even length. + // [0 0] is [tag=0/wiretype=varint varint-encoded-0]. + u.setTag(0, zeroField, func(b []byte, f pointer, w int) ([]byte, error) { + return nil, fmt.Errorf("proto: %s: illegal tag 0 (wire type %d)", t, w) + }, 0) + + // Set mask for required field check. + u.reqMask = uint64(1)<= 0 && (tag < 16 || tag < 2*n) { // TODO: what are the right numbers here? + for len(u.dense) <= tag { + u.dense = append(u.dense, unmarshalFieldInfo{}) + } + u.dense[tag] = i + return + } + if u.sparse == nil { + u.sparse = map[uint64]unmarshalFieldInfo{} + } + u.sparse[uint64(tag)] = i +} + +// fieldUnmarshaler returns an unmarshaler for the given field. +func fieldUnmarshaler(f *reflect.StructField) unmarshaler { + if f.Type.Kind() == reflect.Map { + return makeUnmarshalMap(f) + } + return typeUnmarshaler(f.Type, f.Tag.Get("protobuf")) +} + +// typeUnmarshaler returns an unmarshaler for the given field type / field tag pair. +func typeUnmarshaler(t reflect.Type, tags string) unmarshaler { + tagArray := strings.Split(tags, ",") + encoding := tagArray[0] + name := "unknown" + for _, tag := range tagArray[3:] { + if strings.HasPrefix(tag, "name=") { + name = tag[5:] + } + } + + // Figure out packaging (pointer, slice, or both) + slice := false + pointer := false + if t.Kind() == reflect.Slice && t.Elem().Kind() != reflect.Uint8 { + slice = true + t = t.Elem() + } + if t.Kind() == reflect.Ptr { + pointer = true + t = t.Elem() + } + + // We'll never have both pointer and slice for basic types. + if pointer && slice && t.Kind() != reflect.Struct { + panic("both pointer and slice for basic type in " + t.Name()) + } + + switch t.Kind() { + case reflect.Bool: + if pointer { + return unmarshalBoolPtr + } + if slice { + return unmarshalBoolSlice + } + return unmarshalBoolValue + case reflect.Int32: + switch encoding { + case "fixed32": + if pointer { + return unmarshalFixedS32Ptr + } + if slice { + return unmarshalFixedS32Slice + } + return unmarshalFixedS32Value + case "varint": + // this could be int32 or enum + if pointer { + return unmarshalInt32Ptr + } + if slice { + return unmarshalInt32Slice + } + return unmarshalInt32Value + case "zigzag32": + if pointer { + return unmarshalSint32Ptr + } + if slice { + return unmarshalSint32Slice + } + return unmarshalSint32Value + } + case reflect.Int64: + switch encoding { + case "fixed64": + if pointer { + return unmarshalFixedS64Ptr + } + if slice { + return unmarshalFixedS64Slice + } + return unmarshalFixedS64Value + case "varint": + if pointer { + return unmarshalInt64Ptr + } + if slice { + return unmarshalInt64Slice + } + return unmarshalInt64Value + case "zigzag64": + if pointer { + return unmarshalSint64Ptr + } + if slice { + return unmarshalSint64Slice + } + return unmarshalSint64Value + } + case reflect.Uint32: + switch encoding { + case "fixed32": + if pointer { + return unmarshalFixed32Ptr + } + if slice { + return unmarshalFixed32Slice + } + return unmarshalFixed32Value + case "varint": + if pointer { + return unmarshalUint32Ptr + } + if slice { + return unmarshalUint32Slice + } + return unmarshalUint32Value + } + case reflect.Uint64: + switch encoding { + case "fixed64": + if pointer { + return unmarshalFixed64Ptr + } + if slice { + return unmarshalFixed64Slice + } + return unmarshalFixed64Value + case "varint": + if pointer { + return unmarshalUint64Ptr + } + if slice { + return unmarshalUint64Slice + } + return unmarshalUint64Value + } + case reflect.Float32: + if pointer { + return unmarshalFloat32Ptr + } + if slice { + return unmarshalFloat32Slice + } + return unmarshalFloat32Value + case reflect.Float64: + if pointer { + return unmarshalFloat64Ptr + } + if slice { + return unmarshalFloat64Slice + } + return unmarshalFloat64Value + case reflect.Map: + panic("map type in typeUnmarshaler in " + t.Name()) + case reflect.Slice: + if pointer { + panic("bad pointer in slice case in " + t.Name()) + } + if slice { + return unmarshalBytesSlice + } + return unmarshalBytesValue + case reflect.String: + if pointer { + return unmarshalStringPtr + } + if slice { + return unmarshalStringSlice + } + return unmarshalStringValue + case reflect.Struct: + // message or group field + if !pointer { + panic(fmt.Sprintf("message/group field %s:%s without pointer", t, encoding)) + } + switch encoding { + case "bytes": + if slice { + return makeUnmarshalMessageSlicePtr(getUnmarshalInfo(t), name) + } + return makeUnmarshalMessagePtr(getUnmarshalInfo(t), name) + case "group": + if slice { + return makeUnmarshalGroupSlicePtr(getUnmarshalInfo(t), name) + } + return makeUnmarshalGroupPtr(getUnmarshalInfo(t), name) + } + } + panic(fmt.Sprintf("unmarshaler not found type:%s encoding:%s", t, encoding)) +} + +// Below are all the unmarshalers for individual fields of various types. + +func unmarshalInt64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x) + *f.toInt64() = v + return b, nil +} + +func unmarshalInt64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x) + *f.toInt64Ptr() = &v + return b, nil +} + +func unmarshalInt64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x) + s := f.toInt64Slice() + *s = append(*s, v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x) + s := f.toInt64Slice() + *s = append(*s, v) + return b, nil +} + +func unmarshalSint64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x>>1) ^ int64(x)<<63>>63 + *f.toInt64() = v + return b, nil +} + +func unmarshalSint64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x>>1) ^ int64(x)<<63>>63 + *f.toInt64Ptr() = &v + return b, nil +} + +func unmarshalSint64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x>>1) ^ int64(x)<<63>>63 + s := f.toInt64Slice() + *s = append(*s, v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int64(x>>1) ^ int64(x)<<63>>63 + s := f.toInt64Slice() + *s = append(*s, v) + return b, nil +} + +func unmarshalUint64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint64(x) + *f.toUint64() = v + return b, nil +} + +func unmarshalUint64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint64(x) + *f.toUint64Ptr() = &v + return b, nil +} + +func unmarshalUint64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint64(x) + s := f.toUint64Slice() + *s = append(*s, v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint64(x) + s := f.toUint64Slice() + *s = append(*s, v) + return b, nil +} + +func unmarshalInt32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x) + *f.toInt32() = v + return b, nil +} + +func unmarshalInt32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x) + f.setInt32Ptr(v) + return b, nil +} + +func unmarshalInt32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x) + f.appendInt32Slice(v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x) + f.appendInt32Slice(v) + return b, nil +} + +func unmarshalSint32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x>>1) ^ int32(x)<<31>>31 + *f.toInt32() = v + return b, nil +} + +func unmarshalSint32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x>>1) ^ int32(x)<<31>>31 + f.setInt32Ptr(v) + return b, nil +} + +func unmarshalSint32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x>>1) ^ int32(x)<<31>>31 + f.appendInt32Slice(v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := int32(x>>1) ^ int32(x)<<31>>31 + f.appendInt32Slice(v) + return b, nil +} + +func unmarshalUint32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint32(x) + *f.toUint32() = v + return b, nil +} + +func unmarshalUint32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint32(x) + *f.toUint32Ptr() = &v + return b, nil +} + +func unmarshalUint32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint32(x) + s := f.toUint32Slice() + *s = append(*s, v) + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + v := uint32(x) + s := f.toUint32Slice() + *s = append(*s, v) + return b, nil +} + +func unmarshalFixed64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + *f.toUint64() = v + return b[8:], nil +} + +func unmarshalFixed64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + *f.toUint64Ptr() = &v + return b[8:], nil +} + +func unmarshalFixed64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + s := f.toUint64Slice() + *s = append(*s, v) + b = b[8:] + } + return res, nil + } + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + s := f.toUint64Slice() + *s = append(*s, v) + return b[8:], nil +} + +func unmarshalFixedS64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 + *f.toInt64() = v + return b[8:], nil +} + +func unmarshalFixedS64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 + *f.toInt64Ptr() = &v + return b[8:], nil +} + +func unmarshalFixedS64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 + s := f.toInt64Slice() + *s = append(*s, v) + b = b[8:] + } + return res, nil + } + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := int64(b[0]) | int64(b[1])<<8 | int64(b[2])<<16 | int64(b[3])<<24 | int64(b[4])<<32 | int64(b[5])<<40 | int64(b[6])<<48 | int64(b[7])<<56 + s := f.toInt64Slice() + *s = append(*s, v) + return b[8:], nil +} + +func unmarshalFixed32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + *f.toUint32() = v + return b[4:], nil +} + +func unmarshalFixed32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + *f.toUint32Ptr() = &v + return b[4:], nil +} + +func unmarshalFixed32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + s := f.toUint32Slice() + *s = append(*s, v) + b = b[4:] + } + return res, nil + } + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + s := f.toUint32Slice() + *s = append(*s, v) + return b[4:], nil +} + +func unmarshalFixedS32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 + *f.toInt32() = v + return b[4:], nil +} + +func unmarshalFixedS32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 + f.setInt32Ptr(v) + return b[4:], nil +} + +func unmarshalFixedS32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 + f.appendInt32Slice(v) + b = b[4:] + } + return res, nil + } + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 | int32(b[3])<<24 + f.appendInt32Slice(v) + return b[4:], nil +} + +func unmarshalBoolValue(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + // Note: any length varint is allowed, even though any sane + // encoder will use one byte. + // See https://github.com/golang/protobuf/issues/76 + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + // TODO: check if x>1? Tests seem to indicate no. + v := x != 0 + *f.toBool() = v + return b[n:], nil +} + +func unmarshalBoolPtr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + v := x != 0 + *f.toBoolPtr() = &v + return b[n:], nil +} + +func unmarshalBoolSlice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + x, n = decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + v := x != 0 + s := f.toBoolSlice() + *s = append(*s, v) + b = b[n:] + } + return res, nil + } + if w != WireVarint { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + v := x != 0 + s := f.toBoolSlice() + *s = append(*s, v) + return b[n:], nil +} + +func unmarshalFloat64Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56) + *f.toFloat64() = v + return b[8:], nil +} + +func unmarshalFloat64Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56) + *f.toFloat64Ptr() = &v + return b[8:], nil +} + +func unmarshalFloat64Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56) + s := f.toFloat64Slice() + *s = append(*s, v) + b = b[8:] + } + return res, nil + } + if w != WireFixed64 { + return b, errInternalBadWireType + } + if len(b) < 8 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float64frombits(uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56) + s := f.toFloat64Slice() + *s = append(*s, v) + return b[8:], nil +} + +func unmarshalFloat32Value(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24) + *f.toFloat32() = v + return b[4:], nil +} + +func unmarshalFloat32Ptr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24) + *f.toFloat32Ptr() = &v + return b[4:], nil +} + +func unmarshalFloat32Slice(b []byte, f pointer, w int) ([]byte, error) { + if w == WireBytes { // packed + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + res := b[x:] + b = b[:x] + for len(b) > 0 { + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24) + s := f.toFloat32Slice() + *s = append(*s, v) + b = b[4:] + } + return res, nil + } + if w != WireFixed32 { + return b, errInternalBadWireType + } + if len(b) < 4 { + return nil, io.ErrUnexpectedEOF + } + v := math.Float32frombits(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24) + s := f.toFloat32Slice() + *s = append(*s, v) + return b[4:], nil +} + +func unmarshalStringValue(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := string(b[:x]) + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + *f.toString() = v + return b[x:], nil +} + +func unmarshalStringPtr(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := string(b[:x]) + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + *f.toStringPtr() = &v + return b[x:], nil +} + +func unmarshalStringSlice(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := string(b[:x]) + if !utf8.ValidString(v) { + return nil, errInvalidUTF8 + } + s := f.toStringSlice() + *s = append(*s, v) + return b[x:], nil +} + +var emptyBuf [0]byte + +func unmarshalBytesValue(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + // The use of append here is a trick which avoids the zeroing + // that would be required if we used a make/copy pair. + // We append to emptyBuf instead of nil because we want + // a non-nil result even when the length is 0. + v := append(emptyBuf[:], b[:x]...) + *f.toBytes() = v + return b[x:], nil +} + +func unmarshalBytesSlice(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := append(emptyBuf[:], b[:x]...) + s := f.toBytesSlice() + *s = append(*s, v) + return b[x:], nil +} + +func makeUnmarshalMessagePtr(sub *unmarshalInfo, name string) unmarshaler { + return func(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + // First read the message field to see if something is there. + // The semantics of multiple submessages are weird. Instead of + // the last one winning (as it is for all other fields), multiple + // submessages are merged. + v := f.getPointer() + if v.isNil() { + v = valToPointer(reflect.New(sub.typ)) + f.setPointer(v) + } + err := sub.unmarshal(v, b[:x]) + if err != nil { + if r, ok := err.(*RequiredNotSetError); ok { + r.field = name + "." + r.field + } else { + return nil, err + } + } + return b[x:], err + } +} + +func makeUnmarshalMessageSlicePtr(sub *unmarshalInfo, name string) unmarshaler { + return func(b []byte, f pointer, w int) ([]byte, error) { + if w != WireBytes { + return b, errInternalBadWireType + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + v := valToPointer(reflect.New(sub.typ)) + err := sub.unmarshal(v, b[:x]) + if err != nil { + if r, ok := err.(*RequiredNotSetError); ok { + r.field = name + "." + r.field + } else { + return nil, err + } + } + f.appendPointer(v) + return b[x:], err + } +} + +func makeUnmarshalGroupPtr(sub *unmarshalInfo, name string) unmarshaler { + return func(b []byte, f pointer, w int) ([]byte, error) { + if w != WireStartGroup { + return b, errInternalBadWireType + } + x, y := findEndGroup(b) + if x < 0 { + return nil, io.ErrUnexpectedEOF + } + v := f.getPointer() + if v.isNil() { + v = valToPointer(reflect.New(sub.typ)) + f.setPointer(v) + } + err := sub.unmarshal(v, b[:x]) + if err != nil { + if r, ok := err.(*RequiredNotSetError); ok { + r.field = name + "." + r.field + } else { + return nil, err + } + } + return b[y:], err + } +} + +func makeUnmarshalGroupSlicePtr(sub *unmarshalInfo, name string) unmarshaler { + return func(b []byte, f pointer, w int) ([]byte, error) { + if w != WireStartGroup { + return b, errInternalBadWireType + } + x, y := findEndGroup(b) + if x < 0 { + return nil, io.ErrUnexpectedEOF + } + v := valToPointer(reflect.New(sub.typ)) + err := sub.unmarshal(v, b[:x]) + if err != nil { + if r, ok := err.(*RequiredNotSetError); ok { + r.field = name + "." + r.field + } else { + return nil, err + } + } + f.appendPointer(v) + return b[y:], err + } +} + +func makeUnmarshalMap(f *reflect.StructField) unmarshaler { + t := f.Type + kt := t.Key() + vt := t.Elem() + unmarshalKey := typeUnmarshaler(kt, f.Tag.Get("protobuf_key")) + unmarshalVal := typeUnmarshaler(vt, f.Tag.Get("protobuf_val")) + return func(b []byte, f pointer, w int) ([]byte, error) { + // The map entry is a submessage. Figure out how big it is. + if w != WireBytes { + return nil, fmt.Errorf("proto: bad wiretype for map field: got %d want %d", w, WireBytes) + } + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + b = b[n:] + if x > uint64(len(b)) { + return nil, io.ErrUnexpectedEOF + } + r := b[x:] // unused data to return + b = b[:x] // data for map entry + + // Note: we could use #keys * #values ~= 200 functions + // to do map decoding without reflection. Probably not worth it. + // Maps will be somewhat slow. Oh well. + + // Read key and value from data. + k := reflect.New(kt) + v := reflect.New(vt) + for len(b) > 0 { + x, n := decodeVarint(b) + if n == 0 { + return nil, io.ErrUnexpectedEOF + } + wire := int(x) & 7 + b = b[n:] + + var err error + switch x >> 3 { + case 1: + b, err = unmarshalKey(b, valToPointer(k), wire) + case 2: + b, err = unmarshalVal(b, valToPointer(v), wire) + default: + err = errInternalBadWireType // skip unknown tag + } + + if err == nil { + continue + } + if err != errInternalBadWireType { + return nil, err + } + + // Skip past unknown fields. + b, err = skipField(b, wire) + if err != nil { + return nil, err + } + } + + // Get map, allocate if needed. + m := f.asPointerTo(t).Elem() // an addressable map[K]T + if m.IsNil() { + m.Set(reflect.MakeMap(t)) + } + + // Insert into map. + m.SetMapIndex(k.Elem(), v.Elem()) + + return r, nil + } +} + +// makeUnmarshalOneof makes an unmarshaler for oneof fields. +// for: +// message Msg { +// oneof F { +// int64 X = 1; +// float64 Y = 2; +// } +// } +// typ is the type of the concrete entry for a oneof case (e.g. Msg_X). +// ityp is the interface type of the oneof field (e.g. isMsg_F). +// unmarshal is the unmarshaler for the base type of the oneof case (e.g. int64). +// Note that this function will be called once for each case in the oneof. +func makeUnmarshalOneof(typ, ityp reflect.Type, unmarshal unmarshaler) unmarshaler { + sf := typ.Field(0) + field0 := toField(&sf) + return func(b []byte, f pointer, w int) ([]byte, error) { + // Allocate holder for value. + v := reflect.New(typ) + + // Unmarshal data into holder. + // We unmarshal into the first field of the holder object. + var err error + b, err = unmarshal(b, valToPointer(v).offset(field0), w) + if err != nil { + return nil, err + } + + // Write pointer to holder into target field. + f.asPointerTo(ityp).Elem().Set(v) + + return b, nil + } +} + +// Error used by decode internally. +var errInternalBadWireType = errors.New("proto: internal error: bad wiretype") + +// skipField skips past a field of type wire and returns the remaining bytes. +func skipField(b []byte, wire int) ([]byte, error) { + switch wire { + case WireVarint: + _, k := decodeVarint(b) + if k == 0 { + return b, io.ErrUnexpectedEOF + } + b = b[k:] + case WireFixed32: + if len(b) < 4 { + return b, io.ErrUnexpectedEOF + } + b = b[4:] + case WireFixed64: + if len(b) < 8 { + return b, io.ErrUnexpectedEOF + } + b = b[8:] + case WireBytes: + m, k := decodeVarint(b) + if k == 0 || uint64(len(b)-k) < m { + return b, io.ErrUnexpectedEOF + } + b = b[uint64(k)+m:] + case WireStartGroup: + _, i := findEndGroup(b) + if i == -1 { + return b, io.ErrUnexpectedEOF + } + b = b[i:] + default: + return b, fmt.Errorf("proto: can't skip unknown wire type %d", wire) + } + return b, nil +} + +// findEndGroup finds the index of the next EndGroup tag. +// Groups may be nested, so the "next" EndGroup tag is the first +// unpaired EndGroup. +// findEndGroup returns the indexes of the start and end of the EndGroup tag. +// Returns (-1,-1) if it can't find one. +func findEndGroup(b []byte) (int, int) { + depth := 1 + i := 0 + for { + x, n := decodeVarint(b[i:]) + if n == 0 { + return -1, -1 + } + j := i + i += n + switch x & 7 { + case WireVarint: + _, k := decodeVarint(b[i:]) + if k == 0 { + return -1, -1 + } + i += k + case WireFixed32: + if len(b)-4 < i { + return -1, -1 + } + i += 4 + case WireFixed64: + if len(b)-8 < i { + return -1, -1 + } + i += 8 + case WireBytes: + m, k := decodeVarint(b[i:]) + if k == 0 { + return -1, -1 + } + i += k + if uint64(len(b)-i) < m { + return -1, -1 + } + i += int(m) + case WireStartGroup: + depth++ + case WireEndGroup: + depth-- + if depth == 0 { + return j, i + } + default: + return -1, -1 + } + } +} + +// encodeVarint appends a varint-encoded integer to b and returns the result. +func encodeVarint(b []byte, x uint64) []byte { + for x >= 1<<7 { + b = append(b, byte(x&0x7f|0x80)) + x >>= 7 + } + return append(b, byte(x)) +} + +// decodeVarint reads a varint-encoded integer from b. +// Returns the decoded integer and the number of bytes read. +// If there is an error, it returns 0,0. +func decodeVarint(b []byte) (uint64, int) { + var x, y uint64 + if len(b) <= 0 { + goto bad + } + x = uint64(b[0]) + if x < 0x80 { + return x, 1 + } + x -= 0x80 + + if len(b) <= 1 { + goto bad + } + y = uint64(b[1]) + x += y << 7 + if y < 0x80 { + return x, 2 + } + x -= 0x80 << 7 + + if len(b) <= 2 { + goto bad + } + y = uint64(b[2]) + x += y << 14 + if y < 0x80 { + return x, 3 + } + x -= 0x80 << 14 + + if len(b) <= 3 { + goto bad + } + y = uint64(b[3]) + x += y << 21 + if y < 0x80 { + return x, 4 + } + x -= 0x80 << 21 + + if len(b) <= 4 { + goto bad + } + y = uint64(b[4]) + x += y << 28 + if y < 0x80 { + return x, 5 + } + x -= 0x80 << 28 + + if len(b) <= 5 { + goto bad + } + y = uint64(b[5]) + x += y << 35 + if y < 0x80 { + return x, 6 + } + x -= 0x80 << 35 + + if len(b) <= 6 { + goto bad + } + y = uint64(b[6]) + x += y << 42 + if y < 0x80 { + return x, 7 + } + x -= 0x80 << 42 + + if len(b) <= 7 { + goto bad + } + y = uint64(b[7]) + x += y << 49 + if y < 0x80 { + return x, 8 + } + x -= 0x80 << 49 + + if len(b) <= 8 { + goto bad + } + y = uint64(b[8]) + x += y << 56 + if y < 0x80 { + return x, 9 + } + x -= 0x80 << 56 + + if len(b) <= 9 { + goto bad + } + y = uint64(b[9]) + x += y << 63 + if y < 2 { + return x, 10 + } + +bad: + return 0, 0 +} diff --git a/vendor/github.com/golang/protobuf/proto/text.go b/vendor/github.com/golang/protobuf/proto/text.go index 965876bf..2205fdaa 100644 --- a/vendor/github.com/golang/protobuf/proto/text.go +++ b/vendor/github.com/golang/protobuf/proto/text.go @@ -50,7 +50,6 @@ import ( var ( newline = []byte("\n") spaces = []byte(" ") - gtNewline = []byte(">\n") endBraceNewline = []byte("}\n") backslashN = []byte{'\\', 'n'} backslashR = []byte{'\\', 'r'} @@ -170,11 +169,6 @@ func writeName(w *textWriter, props *Properties) error { return nil } -// raw is the interface satisfied by RawMessage. -type raw interface { - Bytes() []byte -} - func requiresQuotes(u string) bool { // When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted. for _, ch := range u { @@ -269,6 +263,10 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { props := sprops.Prop[i] name := st.Field(i).Name + if name == "XXX_NoUnkeyedLiteral" { + continue + } + if strings.HasPrefix(name, "XXX_") { // There are two XXX_ fields: // XXX_unrecognized []byte @@ -436,12 +434,6 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { return err } } - if b, ok := fv.Interface().(raw); ok { - if err := writeRaw(w, b.Bytes()); err != nil { - return err - } - continue - } // Enums have a String method, so writeAny will work fine. if err := tm.writeAny(w, fv, props); err != nil { @@ -455,7 +447,7 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { // Extensions (the XXX_extensions field). pv := sv.Addr() - if _, ok := extendable(pv.Interface()); ok { + if _, err := extendable(pv.Interface()); err == nil { if err := tm.writeExtensions(w, pv); err != nil { return err } @@ -464,27 +456,6 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { return nil } -// writeRaw writes an uninterpreted raw message. -func writeRaw(w *textWriter, b []byte) error { - if err := w.WriteByte('<'); err != nil { - return err - } - if !w.compact { - if err := w.WriteByte('\n'); err != nil { - return err - } - } - w.indent() - if err := writeUnknownStruct(w, b); err != nil { - return err - } - w.unindent() - if err := w.WriteByte('>'); err != nil { - return err - } - return nil -} - // writeAny writes an arbitrary field. func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error { v = reflect.Indirect(v) @@ -535,6 +506,19 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert } } w.indent() + if v.CanAddr() { + // Calling v.Interface on a struct causes the reflect package to + // copy the entire struct. This is racy with the new Marshaler + // since we atomically update the XXX_sizecache. + // + // Thus, we retrieve a pointer to the struct if possible to avoid + // a race since v.Interface on the pointer doesn't copy the struct. + // + // If v is not addressable, then we are not worried about a race + // since it implies that the binary Marshaler cannot possibly be + // mutating this value. + v = v.Addr() + } if etm, ok := v.Interface().(encoding.TextMarshaler); ok { text, err := etm.MarshalText() if err != nil { @@ -543,8 +527,13 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert if _, err = w.Write(text); err != nil { return err } - } else if err := tm.writeStruct(w, v); err != nil { - return err + } else { + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if err := tm.writeStruct(w, v); err != nil { + return err + } } w.unindent() if err := w.WriteByte(ket); err != nil { diff --git a/vendor/github.com/golang/protobuf/proto/text_parser.go b/vendor/github.com/golang/protobuf/proto/text_parser.go index 61f83c1e..0685bae3 100644 --- a/vendor/github.com/golang/protobuf/proto/text_parser.go +++ b/vendor/github.com/golang/protobuf/proto/text_parser.go @@ -206,7 +206,6 @@ func (p *textParser) advance() { var ( errBadUTF8 = errors.New("proto: bad UTF-8") - errBadHex = errors.New("proto: bad hexadecimal") ) func unquoteC(s string, quote rune) (string, error) { @@ -277,60 +276,47 @@ func unescape(s string) (ch string, tail string, err error) { return "?", s, nil // trigraph workaround case '\'', '"', '\\': return string(r), s, nil - case '0', '1', '2', '3', '4', '5', '6', '7', 'x', 'X': + case '0', '1', '2', '3', '4', '5', '6', '7': if len(s) < 2 { return "", "", fmt.Errorf(`\%c requires 2 following digits`, r) } - base := 8 - ss := s[:2] + ss := string(r) + s[:2] s = s[2:] - if r == 'x' || r == 'X' { - base = 16 - } else { - ss = string(r) + ss - } - i, err := strconv.ParseUint(ss, base, 8) + i, err := strconv.ParseUint(ss, 8, 8) if err != nil { - return "", "", err + return "", "", fmt.Errorf(`\%s contains non-octal digits`, ss) } return string([]byte{byte(i)}), s, nil - case 'u', 'U': - n := 4 - if r == 'U' { + case 'x', 'X', 'u', 'U': + var n int + switch r { + case 'x', 'X': + n = 2 + case 'u': + n = 4 + case 'U': n = 8 } if len(s) < n { - return "", "", fmt.Errorf(`\%c requires %d digits`, r, n) - } - - bs := make([]byte, n/2) - for i := 0; i < n; i += 2 { - a, ok1 := unhex(s[i]) - b, ok2 := unhex(s[i+1]) - if !ok1 || !ok2 { - return "", "", errBadHex - } - bs[i/2] = a<<4 | b + return "", "", fmt.Errorf(`\%c requires %d following digits`, r, n) } + ss := s[:n] s = s[n:] - return string(bs), s, nil + i, err := strconv.ParseUint(ss, 16, 64) + if err != nil { + return "", "", fmt.Errorf(`\%c%s contains non-hexadecimal digits`, r, ss) + } + if r == 'x' || r == 'X' { + return string([]byte{byte(i)}), s, nil + } + if i > utf8.MaxRune { + return "", "", fmt.Errorf(`\%c%s is not a valid Unicode code point`, r, ss) + } + return string(i), s, nil } return "", "", fmt.Errorf(`unknown escape \%c`, r) } -// Adapted from src/pkg/strconv/quote.go. -func unhex(b byte) (v byte, ok bool) { - switch { - case '0' <= b && b <= '9': - return b - '0', true - case 'a' <= b && b <= 'f': - return b - 'a' + 10, true - case 'A' <= b && b <= 'F': - return b - 'A' + 10, true - } - return 0, false -} - // Back off the parser by one token. Can only be done between calls to next(). // It makes the next advance() a no-op. func (p *textParser) back() { p.backed = true } @@ -728,6 +714,9 @@ func (p *textParser) consumeExtName() (string, error) { if tok.err != nil { return "", p.errorf("unrecognized type_url or extension name: %s", tok.err) } + if p.done && tok.value != "]" { + return "", p.errorf("unclosed type_url or extension name") + } } return strings.Join(parts, ""), nil } @@ -883,13 +872,9 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error { // UnmarshalText returns *RequiredNotSetError. func UnmarshalText(s string, pb Message) error { if um, ok := pb.(encoding.TextUnmarshaler); ok { - err := um.UnmarshalText([]byte(s)) - return err + return um.UnmarshalText([]byte(s)) } pb.Reset() v := reflect.ValueOf(pb) - if pe := newTextParser(s).readStruct(v.Elem(), ""); pe != nil { - return pe - } - return nil + return newTextParser(s).readStruct(v.Elem(), "") } diff --git a/vendor/github.com/golang/snappy/AUTHORS b/vendor/github.com/golang/snappy/AUTHORS new file mode 100644 index 00000000..bcfa1952 --- /dev/null +++ b/vendor/github.com/golang/snappy/AUTHORS @@ -0,0 +1,15 @@ +# This is the official list of Snappy-Go authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as +# Name or Organization +# The email address is not required for organizations. + +# Please keep the list sorted. + +Damian Gryski +Google Inc. +Jan Mercl <0xjnml@gmail.com> +Rodolfo Carvalho +Sebastien Binet diff --git a/vendor/github.com/golang/snappy/CONTRIBUTORS b/vendor/github.com/golang/snappy/CONTRIBUTORS new file mode 100644 index 00000000..931ae316 --- /dev/null +++ b/vendor/github.com/golang/snappy/CONTRIBUTORS @@ -0,0 +1,37 @@ +# This is the official list of people who can contribute +# (and typically have contributed) code to the Snappy-Go repository. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# The submission process automatically checks to make sure +# that people submitting code are listed in this file (by email address). +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# http://code.google.com/legal/individual-cla-v1.0.html +# http://code.google.com/legal/corporate-cla-v1.0.html +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. + +# Names should be added to this file like so: +# Name + +# Please keep the list sorted. + +Damian Gryski +Jan Mercl <0xjnml@gmail.com> +Kai Backman +Marc-Antoine Ruel +Nigel Tao +Rob Pike +Rodolfo Carvalho +Russ Cox +Sebastien Binet diff --git a/vendor/github.com/golang/snappy/LICENSE b/vendor/github.com/golang/snappy/LICENSE new file mode 100644 index 00000000..6050c10f --- /dev/null +++ b/vendor/github.com/golang/snappy/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/golang/snappy/README b/vendor/github.com/golang/snappy/README new file mode 100644 index 00000000..cea12879 --- /dev/null +++ b/vendor/github.com/golang/snappy/README @@ -0,0 +1,107 @@ +The Snappy compression format in the Go programming language. + +To download and install from source: +$ go get github.com/golang/snappy + +Unless otherwise noted, the Snappy-Go source files are distributed +under the BSD-style license found in the LICENSE file. + + + +Benchmarks. + +The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten +or so files, the same set used by the C++ Snappy code (github.com/google/snappy +and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @ +3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29: + +"go test -test.bench=." + +_UFlat0-8 2.19GB/s ± 0% html +_UFlat1-8 1.41GB/s ± 0% urls +_UFlat2-8 23.5GB/s ± 2% jpg +_UFlat3-8 1.91GB/s ± 0% jpg_200 +_UFlat4-8 14.0GB/s ± 1% pdf +_UFlat5-8 1.97GB/s ± 0% html4 +_UFlat6-8 814MB/s ± 0% txt1 +_UFlat7-8 785MB/s ± 0% txt2 +_UFlat8-8 857MB/s ± 0% txt3 +_UFlat9-8 719MB/s ± 1% txt4 +_UFlat10-8 2.84GB/s ± 0% pb +_UFlat11-8 1.05GB/s ± 0% gaviota + +_ZFlat0-8 1.04GB/s ± 0% html +_ZFlat1-8 534MB/s ± 0% urls +_ZFlat2-8 15.7GB/s ± 1% jpg +_ZFlat3-8 740MB/s ± 3% jpg_200 +_ZFlat4-8 9.20GB/s ± 1% pdf +_ZFlat5-8 991MB/s ± 0% html4 +_ZFlat6-8 379MB/s ± 0% txt1 +_ZFlat7-8 352MB/s ± 0% txt2 +_ZFlat8-8 396MB/s ± 1% txt3 +_ZFlat9-8 327MB/s ± 1% txt4 +_ZFlat10-8 1.33GB/s ± 1% pb +_ZFlat11-8 605MB/s ± 1% gaviota + + + +"go test -test.bench=. -tags=noasm" + +_UFlat0-8 621MB/s ± 2% html +_UFlat1-8 494MB/s ± 1% urls +_UFlat2-8 23.2GB/s ± 1% jpg +_UFlat3-8 1.12GB/s ± 1% jpg_200 +_UFlat4-8 4.35GB/s ± 1% pdf +_UFlat5-8 609MB/s ± 0% html4 +_UFlat6-8 296MB/s ± 0% txt1 +_UFlat7-8 288MB/s ± 0% txt2 +_UFlat8-8 309MB/s ± 1% txt3 +_UFlat9-8 280MB/s ± 1% txt4 +_UFlat10-8 753MB/s ± 0% pb +_UFlat11-8 400MB/s ± 0% gaviota + +_ZFlat0-8 409MB/s ± 1% html +_ZFlat1-8 250MB/s ± 1% urls +_ZFlat2-8 12.3GB/s ± 1% jpg +_ZFlat3-8 132MB/s ± 0% jpg_200 +_ZFlat4-8 2.92GB/s ± 0% pdf +_ZFlat5-8 405MB/s ± 1% html4 +_ZFlat6-8 179MB/s ± 1% txt1 +_ZFlat7-8 170MB/s ± 1% txt2 +_ZFlat8-8 189MB/s ± 1% txt3 +_ZFlat9-8 164MB/s ± 1% txt4 +_ZFlat10-8 479MB/s ± 1% pb +_ZFlat11-8 270MB/s ± 1% gaviota + + + +For comparison (Go's encoded output is byte-for-byte identical to C++'s), here +are the numbers from C++ Snappy's + +make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log + +BM_UFlat/0 2.4GB/s html +BM_UFlat/1 1.4GB/s urls +BM_UFlat/2 21.8GB/s jpg +BM_UFlat/3 1.5GB/s jpg_200 +BM_UFlat/4 13.3GB/s pdf +BM_UFlat/5 2.1GB/s html4 +BM_UFlat/6 1.0GB/s txt1 +BM_UFlat/7 959.4MB/s txt2 +BM_UFlat/8 1.0GB/s txt3 +BM_UFlat/9 864.5MB/s txt4 +BM_UFlat/10 2.9GB/s pb +BM_UFlat/11 1.2GB/s gaviota + +BM_ZFlat/0 944.3MB/s html (22.31 %) +BM_ZFlat/1 501.6MB/s urls (47.78 %) +BM_ZFlat/2 14.3GB/s jpg (99.95 %) +BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %) +BM_ZFlat/4 8.3GB/s pdf (83.30 %) +BM_ZFlat/5 903.5MB/s html4 (22.52 %) +BM_ZFlat/6 336.0MB/s txt1 (57.88 %) +BM_ZFlat/7 312.3MB/s txt2 (61.91 %) +BM_ZFlat/8 353.1MB/s txt3 (54.99 %) +BM_ZFlat/9 289.9MB/s txt4 (66.26 %) +BM_ZFlat/10 1.2GB/s pb (19.68 %) +BM_ZFlat/11 527.4MB/s gaviota (37.72 %) diff --git a/vendor/github.com/golang/snappy/decode.go b/vendor/github.com/golang/snappy/decode.go new file mode 100644 index 00000000..72efb035 --- /dev/null +++ b/vendor/github.com/golang/snappy/decode.go @@ -0,0 +1,237 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" + "errors" + "io" +) + +var ( + // ErrCorrupt reports that the input is invalid. + ErrCorrupt = errors.New("snappy: corrupt input") + // ErrTooLarge reports that the uncompressed length is too large. + ErrTooLarge = errors.New("snappy: decoded block is too large") + // ErrUnsupported reports that the input isn't supported. + ErrUnsupported = errors.New("snappy: unsupported input") + + errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length") +) + +// DecodedLen returns the length of the decoded block. +func DecodedLen(src []byte) (int, error) { + v, _, err := decodedLen(src) + return v, err +} + +// decodedLen returns the length of the decoded block and the number of bytes +// that the length header occupied. +func decodedLen(src []byte) (blockLen, headerLen int, err error) { + v, n := binary.Uvarint(src) + if n <= 0 || v > 0xffffffff { + return 0, 0, ErrCorrupt + } + + const wordSize = 32 << (^uint(0) >> 32 & 1) + if wordSize == 32 && v > 0x7fffffff { + return 0, 0, ErrTooLarge + } + return int(v), n, nil +} + +const ( + decodeErrCodeCorrupt = 1 + decodeErrCodeUnsupportedLiteralLength = 2 +) + +// Decode returns the decoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire decoded block. +// Otherwise, a newly allocated slice will be returned. +// +// The dst and src must not overlap. It is valid to pass a nil dst. +func Decode(dst, src []byte) ([]byte, error) { + dLen, s, err := decodedLen(src) + if err != nil { + return nil, err + } + if dLen <= len(dst) { + dst = dst[:dLen] + } else { + dst = make([]byte, dLen) + } + switch decode(dst, src[s:]) { + case 0: + return dst, nil + case decodeErrCodeUnsupportedLiteralLength: + return nil, errUnsupportedLiteralLength + } + return nil, ErrCorrupt +} + +// NewReader returns a new Reader that decompresses from r, using the framing +// format described at +// https://github.com/google/snappy/blob/master/framing_format.txt +func NewReader(r io.Reader) *Reader { + return &Reader{ + r: r, + decoded: make([]byte, maxBlockSize), + buf: make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize), + } +} + +// Reader is an io.Reader that can read Snappy-compressed bytes. +type Reader struct { + r io.Reader + err error + decoded []byte + buf []byte + // decoded[i:j] contains decoded bytes that have not yet been passed on. + i, j int + readHeader bool +} + +// Reset discards any buffered data, resets all state, and switches the Snappy +// reader to read from r. This permits reusing a Reader rather than allocating +// a new one. +func (r *Reader) Reset(reader io.Reader) { + r.r = reader + r.err = nil + r.i = 0 + r.j = 0 + r.readHeader = false +} + +func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) { + if _, r.err = io.ReadFull(r.r, p); r.err != nil { + if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { + r.err = ErrCorrupt + } + return false + } + return true +} + +// Read satisfies the io.Reader interface. +func (r *Reader) Read(p []byte) (int, error) { + if r.err != nil { + return 0, r.err + } + for { + if r.i < r.j { + n := copy(p, r.decoded[r.i:r.j]) + r.i += n + return n, nil + } + if !r.readFull(r.buf[:4], true) { + return 0, r.err + } + chunkType := r.buf[0] + if !r.readHeader { + if chunkType != chunkTypeStreamIdentifier { + r.err = ErrCorrupt + return 0, r.err + } + r.readHeader = true + } + chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 + if chunkLen > len(r.buf) { + r.err = ErrUnsupported + return 0, r.err + } + + // The chunk types are specified at + // https://github.com/google/snappy/blob/master/framing_format.txt + switch chunkType { + case chunkTypeCompressedData: + // Section 4.2. Compressed data (chunk type 0x00). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:chunkLen] + if !r.readFull(buf, false) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + buf = buf[checksumSize:] + + n, err := DecodedLen(buf) + if err != nil { + r.err = err + return 0, r.err + } + if n > len(r.decoded) { + r.err = ErrCorrupt + return 0, r.err + } + if _, err := Decode(r.decoded, buf); err != nil { + r.err = err + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeUncompressedData: + // Section 4.3. Uncompressed data (chunk type 0x01). + if chunkLen < checksumSize { + r.err = ErrCorrupt + return 0, r.err + } + buf := r.buf[:checksumSize] + if !r.readFull(buf, false) { + return 0, r.err + } + checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 + // Read directly into r.decoded instead of via r.buf. + n := chunkLen - checksumSize + if n > len(r.decoded) { + r.err = ErrCorrupt + return 0, r.err + } + if !r.readFull(r.decoded[:n], false) { + return 0, r.err + } + if crc(r.decoded[:n]) != checksum { + r.err = ErrCorrupt + return 0, r.err + } + r.i, r.j = 0, n + continue + + case chunkTypeStreamIdentifier: + // Section 4.1. Stream identifier (chunk type 0xff). + if chunkLen != len(magicBody) { + r.err = ErrCorrupt + return 0, r.err + } + if !r.readFull(r.buf[:len(magicBody)], false) { + return 0, r.err + } + for i := 0; i < len(magicBody); i++ { + if r.buf[i] != magicBody[i] { + r.err = ErrCorrupt + return 0, r.err + } + } + continue + } + + if chunkType <= 0x7f { + // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). + r.err = ErrUnsupported + return 0, r.err + } + // Section 4.4 Padding (chunk type 0xfe). + // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). + if !r.readFull(r.buf[:chunkLen], false) { + return 0, r.err + } + } +} diff --git a/vendor/github.com/golang/snappy/decode_amd64.go b/vendor/github.com/golang/snappy/decode_amd64.go new file mode 100644 index 00000000..fcd192b8 --- /dev/null +++ b/vendor/github.com/golang/snappy/decode_amd64.go @@ -0,0 +1,14 @@ +// Copyright 2016 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +package snappy + +// decode has the same semantics as in decode_other.go. +// +//go:noescape +func decode(dst, src []byte) int diff --git a/vendor/github.com/golang/snappy/decode_amd64.s b/vendor/github.com/golang/snappy/decode_amd64.s new file mode 100644 index 00000000..e6179f65 --- /dev/null +++ b/vendor/github.com/golang/snappy/decode_amd64.s @@ -0,0 +1,490 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +#include "textflag.h" + +// The asm code generally follows the pure Go code in decode_other.go, except +// where marked with a "!!!". + +// func decode(dst, src []byte) int +// +// All local variables fit into registers. The non-zero stack size is only to +// spill registers and push args when issuing a CALL. The register allocation: +// - AX scratch +// - BX scratch +// - CX length or x +// - DX offset +// - SI &src[s] +// - DI &dst[d] +// + R8 dst_base +// + R9 dst_len +// + R10 dst_base + dst_len +// + R11 src_base +// + R12 src_len +// + R13 src_base + src_len +// - R14 used by doCopy +// - R15 used by doCopy +// +// The registers R8-R13 (marked with a "+") are set at the start of the +// function, and after a CALL returns, and are not otherwise modified. +// +// The d variable is implicitly DI - R8, and len(dst)-d is R10 - DI. +// The s variable is implicitly SI - R11, and len(src)-s is R13 - SI. +TEXT ·decode(SB), NOSPLIT, $48-56 + // Initialize SI, DI and R8-R13. + MOVQ dst_base+0(FP), R8 + MOVQ dst_len+8(FP), R9 + MOVQ R8, DI + MOVQ R8, R10 + ADDQ R9, R10 + MOVQ src_base+24(FP), R11 + MOVQ src_len+32(FP), R12 + MOVQ R11, SI + MOVQ R11, R13 + ADDQ R12, R13 + +loop: + // for s < len(src) + CMPQ SI, R13 + JEQ end + + // CX = uint32(src[s]) + // + // switch src[s] & 0x03 + MOVBLZX (SI), CX + MOVL CX, BX + ANDL $3, BX + CMPL BX, $1 + JAE tagCopy + + // ---------------------------------------- + // The code below handles literal tags. + + // case tagLiteral: + // x := uint32(src[s] >> 2) + // switch + SHRL $2, CX + CMPL CX, $60 + JAE tagLit60Plus + + // case x < 60: + // s++ + INCQ SI + +doLit: + // This is the end of the inner "switch", when we have a literal tag. + // + // We assume that CX == x and x fits in a uint32, where x is the variable + // used in the pure Go decode_other.go code. + + // length = int(x) + 1 + // + // Unlike the pure Go code, we don't need to check if length <= 0 because + // CX can hold 64 bits, so the increment cannot overflow. + INCQ CX + + // Prepare to check if copying length bytes will run past the end of dst or + // src. + // + // AX = len(dst) - d + // BX = len(src) - s + MOVQ R10, AX + SUBQ DI, AX + MOVQ R13, BX + SUBQ SI, BX + + // !!! Try a faster technique for short (16 or fewer bytes) copies. + // + // if length > 16 || len(dst)-d < 16 || len(src)-s < 16 { + // goto callMemmove // Fall back on calling runtime·memmove. + // } + // + // The C++ snappy code calls this TryFastAppend. It also checks len(src)-s + // against 21 instead of 16, because it cannot assume that all of its input + // is contiguous in memory and so it needs to leave enough source bytes to + // read the next tag without refilling buffers, but Go's Decode assumes + // contiguousness (the src argument is a []byte). + CMPQ CX, $16 + JGT callMemmove + CMPQ AX, $16 + JLT callMemmove + CMPQ BX, $16 + JLT callMemmove + + // !!! Implement the copy from src to dst as a 16-byte load and store. + // (Decode's documentation says that dst and src must not overlap.) + // + // This always copies 16 bytes, instead of only length bytes, but that's + // OK. If the input is a valid Snappy encoding then subsequent iterations + // will fix up the overrun. Otherwise, Decode returns a nil []byte (and a + // non-nil error), so the overrun will be ignored. + // + // Note that on amd64, it is legal and cheap to issue unaligned 8-byte or + // 16-byte loads and stores. This technique probably wouldn't be as + // effective on architectures that are fussier about alignment. + MOVOU 0(SI), X0 + MOVOU X0, 0(DI) + + // d += length + // s += length + ADDQ CX, DI + ADDQ CX, SI + JMP loop + +callMemmove: + // if length > len(dst)-d || length > len(src)-s { etc } + CMPQ CX, AX + JGT errCorrupt + CMPQ CX, BX + JGT errCorrupt + + // copy(dst[d:], src[s:s+length]) + // + // This means calling runtime·memmove(&dst[d], &src[s], length), so we push + // DI, SI and CX as arguments. Coincidentally, we also need to spill those + // three registers to the stack, to save local variables across the CALL. + MOVQ DI, 0(SP) + MOVQ SI, 8(SP) + MOVQ CX, 16(SP) + MOVQ DI, 24(SP) + MOVQ SI, 32(SP) + MOVQ CX, 40(SP) + CALL runtime·memmove(SB) + + // Restore local variables: unspill registers from the stack and + // re-calculate R8-R13. + MOVQ 24(SP), DI + MOVQ 32(SP), SI + MOVQ 40(SP), CX + MOVQ dst_base+0(FP), R8 + MOVQ dst_len+8(FP), R9 + MOVQ R8, R10 + ADDQ R9, R10 + MOVQ src_base+24(FP), R11 + MOVQ src_len+32(FP), R12 + MOVQ R11, R13 + ADDQ R12, R13 + + // d += length + // s += length + ADDQ CX, DI + ADDQ CX, SI + JMP loop + +tagLit60Plus: + // !!! This fragment does the + // + // s += x - 58; if uint(s) > uint(len(src)) { etc } + // + // checks. In the asm version, we code it once instead of once per switch case. + ADDQ CX, SI + SUBQ $58, SI + MOVQ SI, BX + SUBQ R11, BX + CMPQ BX, R12 + JA errCorrupt + + // case x == 60: + CMPL CX, $61 + JEQ tagLit61 + JA tagLit62Plus + + // x = uint32(src[s-1]) + MOVBLZX -1(SI), CX + JMP doLit + +tagLit61: + // case x == 61: + // x = uint32(src[s-2]) | uint32(src[s-1])<<8 + MOVWLZX -2(SI), CX + JMP doLit + +tagLit62Plus: + CMPL CX, $62 + JA tagLit63 + + // case x == 62: + // x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 + MOVWLZX -3(SI), CX + MOVBLZX -1(SI), BX + SHLL $16, BX + ORL BX, CX + JMP doLit + +tagLit63: + // case x == 63: + // x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 + MOVL -4(SI), CX + JMP doLit + +// The code above handles literal tags. +// ---------------------------------------- +// The code below handles copy tags. + +tagCopy4: + // case tagCopy4: + // s += 5 + ADDQ $5, SI + + // if uint(s) > uint(len(src)) { etc } + MOVQ SI, BX + SUBQ R11, BX + CMPQ BX, R12 + JA errCorrupt + + // length = 1 + int(src[s-5])>>2 + SHRQ $2, CX + INCQ CX + + // offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) + MOVLQZX -4(SI), DX + JMP doCopy + +tagCopy2: + // case tagCopy2: + // s += 3 + ADDQ $3, SI + + // if uint(s) > uint(len(src)) { etc } + MOVQ SI, BX + SUBQ R11, BX + CMPQ BX, R12 + JA errCorrupt + + // length = 1 + int(src[s-3])>>2 + SHRQ $2, CX + INCQ CX + + // offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) + MOVWQZX -2(SI), DX + JMP doCopy + +tagCopy: + // We have a copy tag. We assume that: + // - BX == src[s] & 0x03 + // - CX == src[s] + CMPQ BX, $2 + JEQ tagCopy2 + JA tagCopy4 + + // case tagCopy1: + // s += 2 + ADDQ $2, SI + + // if uint(s) > uint(len(src)) { etc } + MOVQ SI, BX + SUBQ R11, BX + CMPQ BX, R12 + JA errCorrupt + + // offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) + MOVQ CX, DX + ANDQ $0xe0, DX + SHLQ $3, DX + MOVBQZX -1(SI), BX + ORQ BX, DX + + // length = 4 + int(src[s-2])>>2&0x7 + SHRQ $2, CX + ANDQ $7, CX + ADDQ $4, CX + +doCopy: + // This is the end of the outer "switch", when we have a copy tag. + // + // We assume that: + // - CX == length && CX > 0 + // - DX == offset + + // if offset <= 0 { etc } + CMPQ DX, $0 + JLE errCorrupt + + // if d < offset { etc } + MOVQ DI, BX + SUBQ R8, BX + CMPQ BX, DX + JLT errCorrupt + + // if length > len(dst)-d { etc } + MOVQ R10, BX + SUBQ DI, BX + CMPQ CX, BX + JGT errCorrupt + + // forwardCopy(dst[d:d+length], dst[d-offset:]); d += length + // + // Set: + // - R14 = len(dst)-d + // - R15 = &dst[d-offset] + MOVQ R10, R14 + SUBQ DI, R14 + MOVQ DI, R15 + SUBQ DX, R15 + + // !!! Try a faster technique for short (16 or fewer bytes) forward copies. + // + // First, try using two 8-byte load/stores, similar to the doLit technique + // above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is + // still OK if offset >= 8. Note that this has to be two 8-byte load/stores + // and not one 16-byte load/store, and the first store has to be before the + // second load, due to the overlap if offset is in the range [8, 16). + // + // if length > 16 || offset < 8 || len(dst)-d < 16 { + // goto slowForwardCopy + // } + // copy 16 bytes + // d += length + CMPQ CX, $16 + JGT slowForwardCopy + CMPQ DX, $8 + JLT slowForwardCopy + CMPQ R14, $16 + JLT slowForwardCopy + MOVQ 0(R15), AX + MOVQ AX, 0(DI) + MOVQ 8(R15), BX + MOVQ BX, 8(DI) + ADDQ CX, DI + JMP loop + +slowForwardCopy: + // !!! If the forward copy is longer than 16 bytes, or if offset < 8, we + // can still try 8-byte load stores, provided we can overrun up to 10 extra + // bytes. As above, the overrun will be fixed up by subsequent iterations + // of the outermost loop. + // + // The C++ snappy code calls this technique IncrementalCopyFastPath. Its + // commentary says: + // + // ---- + // + // The main part of this loop is a simple copy of eight bytes at a time + // until we've copied (at least) the requested amount of bytes. However, + // if d and d-offset are less than eight bytes apart (indicating a + // repeating pattern of length < 8), we first need to expand the pattern in + // order to get the correct results. For instance, if the buffer looks like + // this, with the eight-byte and patterns marked as + // intervals: + // + // abxxxxxxxxxxxx + // [------] d-offset + // [------] d + // + // a single eight-byte copy from to will repeat the pattern + // once, after which we can move two bytes without moving : + // + // ababxxxxxxxxxx + // [------] d-offset + // [------] d + // + // and repeat the exercise until the two no longer overlap. + // + // This allows us to do very well in the special case of one single byte + // repeated many times, without taking a big hit for more general cases. + // + // The worst case of extra writing past the end of the match occurs when + // offset == 1 and length == 1; the last copy will read from byte positions + // [0..7] and write to [4..11], whereas it was only supposed to write to + // position 1. Thus, ten excess bytes. + // + // ---- + // + // That "10 byte overrun" worst case is confirmed by Go's + // TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy + // and finishSlowForwardCopy algorithm. + // + // if length > len(dst)-d-10 { + // goto verySlowForwardCopy + // } + SUBQ $10, R14 + CMPQ CX, R14 + JGT verySlowForwardCopy + +makeOffsetAtLeast8: + // !!! As above, expand the pattern so that offset >= 8 and we can use + // 8-byte load/stores. + // + // for offset < 8 { + // copy 8 bytes from dst[d-offset:] to dst[d:] + // length -= offset + // d += offset + // offset += offset + // // The two previous lines together means that d-offset, and therefore + // // R15, is unchanged. + // } + CMPQ DX, $8 + JGE fixUpSlowForwardCopy + MOVQ (R15), BX + MOVQ BX, (DI) + SUBQ DX, CX + ADDQ DX, DI + ADDQ DX, DX + JMP makeOffsetAtLeast8 + +fixUpSlowForwardCopy: + // !!! Add length (which might be negative now) to d (implied by DI being + // &dst[d]) so that d ends up at the right place when we jump back to the + // top of the loop. Before we do that, though, we save DI to AX so that, if + // length is positive, copying the remaining length bytes will write to the + // right place. + MOVQ DI, AX + ADDQ CX, DI + +finishSlowForwardCopy: + // !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative + // length means that we overrun, but as above, that will be fixed up by + // subsequent iterations of the outermost loop. + CMPQ CX, $0 + JLE loop + MOVQ (R15), BX + MOVQ BX, (AX) + ADDQ $8, R15 + ADDQ $8, AX + SUBQ $8, CX + JMP finishSlowForwardCopy + +verySlowForwardCopy: + // verySlowForwardCopy is a simple implementation of forward copy. In C + // parlance, this is a do/while loop instead of a while loop, since we know + // that length > 0. In Go syntax: + // + // for { + // dst[d] = dst[d - offset] + // d++ + // length-- + // if length == 0 { + // break + // } + // } + MOVB (R15), BX + MOVB BX, (DI) + INCQ R15 + INCQ DI + DECQ CX + JNZ verySlowForwardCopy + JMP loop + +// The code above handles copy tags. +// ---------------------------------------- + +end: + // This is the end of the "for s < len(src)". + // + // if d != len(dst) { etc } + CMPQ DI, R10 + JNE errCorrupt + + // return 0 + MOVQ $0, ret+48(FP) + RET + +errCorrupt: + // return decodeErrCodeCorrupt + MOVQ $1, ret+48(FP) + RET diff --git a/vendor/github.com/golang/snappy/decode_other.go b/vendor/github.com/golang/snappy/decode_other.go new file mode 100644 index 00000000..8c9f2049 --- /dev/null +++ b/vendor/github.com/golang/snappy/decode_other.go @@ -0,0 +1,101 @@ +// Copyright 2016 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !amd64 appengine !gc noasm + +package snappy + +// decode writes the decoding of src to dst. It assumes that the varint-encoded +// length of the decompressed bytes has already been read, and that len(dst) +// equals that length. +// +// It returns 0 on success or a decodeErrCodeXxx error code on failure. +func decode(dst, src []byte) int { + var d, s, offset, length int + for s < len(src) { + switch src[s] & 0x03 { + case tagLiteral: + x := uint32(src[s] >> 2) + switch { + case x < 60: + s++ + case x == 60: + s += 2 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + x = uint32(src[s-1]) + case x == 61: + s += 3 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + x = uint32(src[s-2]) | uint32(src[s-1])<<8 + case x == 62: + s += 4 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 + case x == 63: + s += 5 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 + } + length = int(x) + 1 + if length <= 0 { + return decodeErrCodeUnsupportedLiteralLength + } + if length > len(dst)-d || length > len(src)-s { + return decodeErrCodeCorrupt + } + copy(dst[d:], src[s:s+length]) + d += length + s += length + continue + + case tagCopy1: + s += 2 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + length = 4 + int(src[s-2])>>2&0x7 + offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) + + case tagCopy2: + s += 3 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + length = 1 + int(src[s-3])>>2 + offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) + + case tagCopy4: + s += 5 + if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. + return decodeErrCodeCorrupt + } + length = 1 + int(src[s-5])>>2 + offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) + } + + if offset <= 0 || d < offset || length > len(dst)-d { + return decodeErrCodeCorrupt + } + // Copy from an earlier sub-slice of dst to a later sub-slice. Unlike + // the built-in copy function, this byte-by-byte copy always runs + // forwards, even if the slices overlap. Conceptually, this is: + // + // d += forwardCopy(dst[d:d+length], dst[d-offset:]) + for end := d + length; d != end; d++ { + dst[d] = dst[d-offset] + } + } + if d != len(dst) { + return decodeErrCodeCorrupt + } + return 0 +} diff --git a/vendor/github.com/golang/snappy/encode.go b/vendor/github.com/golang/snappy/encode.go new file mode 100644 index 00000000..8d393e90 --- /dev/null +++ b/vendor/github.com/golang/snappy/encode.go @@ -0,0 +1,285 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package snappy + +import ( + "encoding/binary" + "errors" + "io" +) + +// Encode returns the encoded form of src. The returned slice may be a sub- +// slice of dst if dst was large enough to hold the entire encoded block. +// Otherwise, a newly allocated slice will be returned. +// +// The dst and src must not overlap. It is valid to pass a nil dst. +func Encode(dst, src []byte) []byte { + if n := MaxEncodedLen(len(src)); n < 0 { + panic(ErrTooLarge) + } else if len(dst) < n { + dst = make([]byte, n) + } + + // The block starts with the varint-encoded length of the decompressed bytes. + d := binary.PutUvarint(dst, uint64(len(src))) + + for len(src) > 0 { + p := src + src = nil + if len(p) > maxBlockSize { + p, src = p[:maxBlockSize], p[maxBlockSize:] + } + if len(p) < minNonLiteralBlockSize { + d += emitLiteral(dst[d:], p) + } else { + d += encodeBlock(dst[d:], p) + } + } + return dst[:d] +} + +// inputMargin is the minimum number of extra input bytes to keep, inside +// encodeBlock's inner loop. On some architectures, this margin lets us +// implement a fast path for emitLiteral, where the copy of short (<= 16 byte) +// literals can be implemented as a single load to and store from a 16-byte +// register. That literal's actual length can be as short as 1 byte, so this +// can copy up to 15 bytes too much, but that's OK as subsequent iterations of +// the encoding loop will fix up the copy overrun, and this inputMargin ensures +// that we don't overrun the dst and src buffers. +const inputMargin = 16 - 1 + +// minNonLiteralBlockSize is the minimum size of the input to encodeBlock that +// could be encoded with a copy tag. This is the minimum with respect to the +// algorithm used by encodeBlock, not a minimum enforced by the file format. +// +// The encoded output must start with at least a 1 byte literal, as there are +// no previous bytes to copy. A minimal (1 byte) copy after that, generated +// from an emitCopy call in encodeBlock's main loop, would require at least +// another inputMargin bytes, for the reason above: we want any emitLiteral +// calls inside encodeBlock's main loop to use the fast path if possible, which +// requires being able to overrun by inputMargin bytes. Thus, +// minNonLiteralBlockSize equals 1 + 1 + inputMargin. +// +// The C++ code doesn't use this exact threshold, but it could, as discussed at +// https://groups.google.com/d/topic/snappy-compression/oGbhsdIJSJ8/discussion +// The difference between Go (2+inputMargin) and C++ (inputMargin) is purely an +// optimization. It should not affect the encoded form. This is tested by +// TestSameEncodingAsCppShortCopies. +const minNonLiteralBlockSize = 1 + 1 + inputMargin + +// MaxEncodedLen returns the maximum length of a snappy block, given its +// uncompressed length. +// +// It will return a negative value if srcLen is too large to encode. +func MaxEncodedLen(srcLen int) int { + n := uint64(srcLen) + if n > 0xffffffff { + return -1 + } + // Compressed data can be defined as: + // compressed := item* literal* + // item := literal* copy + // + // The trailing literal sequence has a space blowup of at most 62/60 + // since a literal of length 60 needs one tag byte + one extra byte + // for length information. + // + // Item blowup is trickier to measure. Suppose the "copy" op copies + // 4 bytes of data. Because of a special check in the encoding code, + // we produce a 4-byte copy only if the offset is < 65536. Therefore + // the copy op takes 3 bytes to encode, and this type of item leads + // to at most the 62/60 blowup for representing literals. + // + // Suppose the "copy" op copies 5 bytes of data. If the offset is big + // enough, it will take 5 bytes to encode the copy op. Therefore the + // worst case here is a one-byte literal followed by a five-byte copy. + // That is, 6 bytes of input turn into 7 bytes of "compressed" data. + // + // This last factor dominates the blowup, so the final estimate is: + n = 32 + n + n/6 + if n > 0xffffffff { + return -1 + } + return int(n) +} + +var errClosed = errors.New("snappy: Writer is closed") + +// NewWriter returns a new Writer that compresses to w. +// +// The Writer returned does not buffer writes. There is no need to Flush or +// Close such a Writer. +// +// Deprecated: the Writer returned is not suitable for many small writes, only +// for few large writes. Use NewBufferedWriter instead, which is efficient +// regardless of the frequency and shape of the writes, and remember to Close +// that Writer when done. +func NewWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + obuf: make([]byte, obufLen), + } +} + +// NewBufferedWriter returns a new Writer that compresses to w, using the +// framing format described at +// https://github.com/google/snappy/blob/master/framing_format.txt +// +// The Writer returned buffers writes. Users must call Close to guarantee all +// data has been forwarded to the underlying io.Writer. They may also call +// Flush zero or more times before calling Close. +func NewBufferedWriter(w io.Writer) *Writer { + return &Writer{ + w: w, + ibuf: make([]byte, 0, maxBlockSize), + obuf: make([]byte, obufLen), + } +} + +// Writer is an io.Writer that can write Snappy-compressed bytes. +type Writer struct { + w io.Writer + err error + + // ibuf is a buffer for the incoming (uncompressed) bytes. + // + // Its use is optional. For backwards compatibility, Writers created by the + // NewWriter function have ibuf == nil, do not buffer incoming bytes, and + // therefore do not need to be Flush'ed or Close'd. + ibuf []byte + + // obuf is a buffer for the outgoing (compressed) bytes. + obuf []byte + + // wroteStreamHeader is whether we have written the stream header. + wroteStreamHeader bool +} + +// Reset discards the writer's state and switches the Snappy writer to write to +// w. This permits reusing a Writer rather than allocating a new one. +func (w *Writer) Reset(writer io.Writer) { + w.w = writer + w.err = nil + if w.ibuf != nil { + w.ibuf = w.ibuf[:0] + } + w.wroteStreamHeader = false +} + +// Write satisfies the io.Writer interface. +func (w *Writer) Write(p []byte) (nRet int, errRet error) { + if w.ibuf == nil { + // Do not buffer incoming bytes. This does not perform or compress well + // if the caller of Writer.Write writes many small slices. This + // behavior is therefore deprecated, but still supported for backwards + // compatibility with code that doesn't explicitly Flush or Close. + return w.write(p) + } + + // The remainder of this method is based on bufio.Writer.Write from the + // standard library. + + for len(p) > (cap(w.ibuf)-len(w.ibuf)) && w.err == nil { + var n int + if len(w.ibuf) == 0 { + // Large write, empty buffer. + // Write directly from p to avoid copy. + n, _ = w.write(p) + } else { + n = copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) + w.ibuf = w.ibuf[:len(w.ibuf)+n] + w.Flush() + } + nRet += n + p = p[n:] + } + if w.err != nil { + return nRet, w.err + } + n := copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) + w.ibuf = w.ibuf[:len(w.ibuf)+n] + nRet += n + return nRet, nil +} + +func (w *Writer) write(p []byte) (nRet int, errRet error) { + if w.err != nil { + return 0, w.err + } + for len(p) > 0 { + obufStart := len(magicChunk) + if !w.wroteStreamHeader { + w.wroteStreamHeader = true + copy(w.obuf, magicChunk) + obufStart = 0 + } + + var uncompressed []byte + if len(p) > maxBlockSize { + uncompressed, p = p[:maxBlockSize], p[maxBlockSize:] + } else { + uncompressed, p = p, nil + } + checksum := crc(uncompressed) + + // Compress the buffer, discarding the result if the improvement + // isn't at least 12.5%. + compressed := Encode(w.obuf[obufHeaderLen:], uncompressed) + chunkType := uint8(chunkTypeCompressedData) + chunkLen := 4 + len(compressed) + obufEnd := obufHeaderLen + len(compressed) + if len(compressed) >= len(uncompressed)-len(uncompressed)/8 { + chunkType = chunkTypeUncompressedData + chunkLen = 4 + len(uncompressed) + obufEnd = obufHeaderLen + } + + // Fill in the per-chunk header that comes before the body. + w.obuf[len(magicChunk)+0] = chunkType + w.obuf[len(magicChunk)+1] = uint8(chunkLen >> 0) + w.obuf[len(magicChunk)+2] = uint8(chunkLen >> 8) + w.obuf[len(magicChunk)+3] = uint8(chunkLen >> 16) + w.obuf[len(magicChunk)+4] = uint8(checksum >> 0) + w.obuf[len(magicChunk)+5] = uint8(checksum >> 8) + w.obuf[len(magicChunk)+6] = uint8(checksum >> 16) + w.obuf[len(magicChunk)+7] = uint8(checksum >> 24) + + if _, err := w.w.Write(w.obuf[obufStart:obufEnd]); err != nil { + w.err = err + return nRet, err + } + if chunkType == chunkTypeUncompressedData { + if _, err := w.w.Write(uncompressed); err != nil { + w.err = err + return nRet, err + } + } + nRet += len(uncompressed) + } + return nRet, nil +} + +// Flush flushes the Writer to its underlying io.Writer. +func (w *Writer) Flush() error { + if w.err != nil { + return w.err + } + if len(w.ibuf) == 0 { + return nil + } + w.write(w.ibuf) + w.ibuf = w.ibuf[:0] + return w.err +} + +// Close calls Flush and then closes the Writer. +func (w *Writer) Close() error { + w.Flush() + ret := w.err + if w.err == nil { + w.err = errClosed + } + return ret +} diff --git a/vendor/github.com/golang/snappy/encode_amd64.go b/vendor/github.com/golang/snappy/encode_amd64.go new file mode 100644 index 00000000..150d91bc --- /dev/null +++ b/vendor/github.com/golang/snappy/encode_amd64.go @@ -0,0 +1,29 @@ +// Copyright 2016 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +package snappy + +// emitLiteral has the same semantics as in encode_other.go. +// +//go:noescape +func emitLiteral(dst, lit []byte) int + +// emitCopy has the same semantics as in encode_other.go. +// +//go:noescape +func emitCopy(dst []byte, offset, length int) int + +// extendMatch has the same semantics as in encode_other.go. +// +//go:noescape +func extendMatch(src []byte, i, j int) int + +// encodeBlock has the same semantics as in encode_other.go. +// +//go:noescape +func encodeBlock(dst, src []byte) (d int) diff --git a/vendor/github.com/golang/snappy/encode_amd64.s b/vendor/github.com/golang/snappy/encode_amd64.s new file mode 100644 index 00000000..adfd979f --- /dev/null +++ b/vendor/github.com/golang/snappy/encode_amd64.s @@ -0,0 +1,730 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +#include "textflag.h" + +// The XXX lines assemble on Go 1.4, 1.5 and 1.7, but not 1.6, due to a +// Go toolchain regression. See https://github.com/golang/go/issues/15426 and +// https://github.com/golang/snappy/issues/29 +// +// As a workaround, the package was built with a known good assembler, and +// those instructions were disassembled by "objdump -d" to yield the +// 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 +// style comments, in AT&T asm syntax. Note that rsp here is a physical +// register, not Go/asm's SP pseudo-register (see https://golang.org/doc/asm). +// The instructions were then encoded as "BYTE $0x.." sequences, which assemble +// fine on Go 1.6. + +// The asm code generally follows the pure Go code in encode_other.go, except +// where marked with a "!!!". + +// ---------------------------------------------------------------------------- + +// func emitLiteral(dst, lit []byte) int +// +// All local variables fit into registers. The register allocation: +// - AX len(lit) +// - BX n +// - DX return value +// - DI &dst[i] +// - R10 &lit[0] +// +// The 24 bytes of stack space is to call runtime·memmove. +// +// The unusual register allocation of local variables, such as R10 for the +// source pointer, matches the allocation used at the call site in encodeBlock, +// which makes it easier to manually inline this function. +TEXT ·emitLiteral(SB), NOSPLIT, $24-56 + MOVQ dst_base+0(FP), DI + MOVQ lit_base+24(FP), R10 + MOVQ lit_len+32(FP), AX + MOVQ AX, DX + MOVL AX, BX + SUBL $1, BX + + CMPL BX, $60 + JLT oneByte + CMPL BX, $256 + JLT twoBytes + +threeBytes: + MOVB $0xf4, 0(DI) + MOVW BX, 1(DI) + ADDQ $3, DI + ADDQ $3, DX + JMP memmove + +twoBytes: + MOVB $0xf0, 0(DI) + MOVB BX, 1(DI) + ADDQ $2, DI + ADDQ $2, DX + JMP memmove + +oneByte: + SHLB $2, BX + MOVB BX, 0(DI) + ADDQ $1, DI + ADDQ $1, DX + +memmove: + MOVQ DX, ret+48(FP) + + // copy(dst[i:], lit) + // + // This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push + // DI, R10 and AX as arguments. + MOVQ DI, 0(SP) + MOVQ R10, 8(SP) + MOVQ AX, 16(SP) + CALL runtime·memmove(SB) + RET + +// ---------------------------------------------------------------------------- + +// func emitCopy(dst []byte, offset, length int) int +// +// All local variables fit into registers. The register allocation: +// - AX length +// - SI &dst[0] +// - DI &dst[i] +// - R11 offset +// +// The unusual register allocation of local variables, such as R11 for the +// offset, matches the allocation used at the call site in encodeBlock, which +// makes it easier to manually inline this function. +TEXT ·emitCopy(SB), NOSPLIT, $0-48 + MOVQ dst_base+0(FP), DI + MOVQ DI, SI + MOVQ offset+24(FP), R11 + MOVQ length+32(FP), AX + +loop0: + // for length >= 68 { etc } + CMPL AX, $68 + JLT step1 + + // Emit a length 64 copy, encoded as 3 bytes. + MOVB $0xfe, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + SUBL $64, AX + JMP loop0 + +step1: + // if length > 64 { etc } + CMPL AX, $64 + JLE step2 + + // Emit a length 60 copy, encoded as 3 bytes. + MOVB $0xee, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + SUBL $60, AX + +step2: + // if length >= 12 || offset >= 2048 { goto step3 } + CMPL AX, $12 + JGE step3 + CMPL R11, $2048 + JGE step3 + + // Emit the remaining copy, encoded as 2 bytes. + MOVB R11, 1(DI) + SHRL $8, R11 + SHLB $5, R11 + SUBB $4, AX + SHLB $2, AX + ORB AX, R11 + ORB $1, R11 + MOVB R11, 0(DI) + ADDQ $2, DI + + // Return the number of bytes written. + SUBQ SI, DI + MOVQ DI, ret+40(FP) + RET + +step3: + // Emit the remaining copy, encoded as 3 bytes. + SUBL $1, AX + SHLB $2, AX + ORB $2, AX + MOVB AX, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + + // Return the number of bytes written. + SUBQ SI, DI + MOVQ DI, ret+40(FP) + RET + +// ---------------------------------------------------------------------------- + +// func extendMatch(src []byte, i, j int) int +// +// All local variables fit into registers. The register allocation: +// - DX &src[0] +// - SI &src[j] +// - R13 &src[len(src) - 8] +// - R14 &src[len(src)] +// - R15 &src[i] +// +// The unusual register allocation of local variables, such as R15 for a source +// pointer, matches the allocation used at the call site in encodeBlock, which +// makes it easier to manually inline this function. +TEXT ·extendMatch(SB), NOSPLIT, $0-48 + MOVQ src_base+0(FP), DX + MOVQ src_len+8(FP), R14 + MOVQ i+24(FP), R15 + MOVQ j+32(FP), SI + ADDQ DX, R14 + ADDQ DX, R15 + ADDQ DX, SI + MOVQ R14, R13 + SUBQ $8, R13 + +cmp8: + // As long as we are 8 or more bytes before the end of src, we can load and + // compare 8 bytes at a time. If those 8 bytes are equal, repeat. + CMPQ SI, R13 + JA cmp1 + MOVQ (R15), AX + MOVQ (SI), BX + CMPQ AX, BX + JNE bsf + ADDQ $8, R15 + ADDQ $8, SI + JMP cmp8 + +bsf: + // If those 8 bytes were not equal, XOR the two 8 byte values, and return + // the index of the first byte that differs. The BSF instruction finds the + // least significant 1 bit, the amd64 architecture is little-endian, and + // the shift by 3 converts a bit index to a byte index. + XORQ AX, BX + BSFQ BX, BX + SHRQ $3, BX + ADDQ BX, SI + + // Convert from &src[ret] to ret. + SUBQ DX, SI + MOVQ SI, ret+40(FP) + RET + +cmp1: + // In src's tail, compare 1 byte at a time. + CMPQ SI, R14 + JAE extendMatchEnd + MOVB (R15), AX + MOVB (SI), BX + CMPB AX, BX + JNE extendMatchEnd + ADDQ $1, R15 + ADDQ $1, SI + JMP cmp1 + +extendMatchEnd: + // Convert from &src[ret] to ret. + SUBQ DX, SI + MOVQ SI, ret+40(FP) + RET + +// ---------------------------------------------------------------------------- + +// func encodeBlock(dst, src []byte) (d int) +// +// All local variables fit into registers, other than "var table". The register +// allocation: +// - AX . . +// - BX . . +// - CX 56 shift (note that amd64 shifts by non-immediates must use CX). +// - DX 64 &src[0], tableSize +// - SI 72 &src[s] +// - DI 80 &dst[d] +// - R9 88 sLimit +// - R10 . &src[nextEmit] +// - R11 96 prevHash, currHash, nextHash, offset +// - R12 104 &src[base], skip +// - R13 . &src[nextS], &src[len(src) - 8] +// - R14 . len(src), bytesBetweenHashLookups, &src[len(src)], x +// - R15 112 candidate +// +// The second column (56, 64, etc) is the stack offset to spill the registers +// when calling other functions. We could pack this slightly tighter, but it's +// simpler to have a dedicated spill map independent of the function called. +// +// "var table [maxTableSize]uint16" takes up 32768 bytes of stack space. An +// extra 56 bytes, to call other functions, and an extra 64 bytes, to spill +// local variables (registers) during calls gives 32768 + 56 + 64 = 32888. +TEXT ·encodeBlock(SB), 0, $32888-56 + MOVQ dst_base+0(FP), DI + MOVQ src_base+24(FP), SI + MOVQ src_len+32(FP), R14 + + // shift, tableSize := uint32(32-8), 1<<8 + MOVQ $24, CX + MOVQ $256, DX + +calcShift: + // for ; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 { + // shift-- + // } + CMPQ DX, $16384 + JGE varTable + CMPQ DX, R14 + JGE varTable + SUBQ $1, CX + SHLQ $1, DX + JMP calcShift + +varTable: + // var table [maxTableSize]uint16 + // + // In the asm code, unlike the Go code, we can zero-initialize only the + // first tableSize elements. Each uint16 element is 2 bytes and each MOVOU + // writes 16 bytes, so we can do only tableSize/8 writes instead of the + // 2048 writes that would zero-initialize all of table's 32768 bytes. + SHRQ $3, DX + LEAQ table-32768(SP), BX + PXOR X0, X0 + +memclr: + MOVOU X0, 0(BX) + ADDQ $16, BX + SUBQ $1, DX + JNZ memclr + + // !!! DX = &src[0] + MOVQ SI, DX + + // sLimit := len(src) - inputMargin + MOVQ R14, R9 + SUBQ $15, R9 + + // !!! Pre-emptively spill CX, DX and R9 to the stack. Their values don't + // change for the rest of the function. + MOVQ CX, 56(SP) + MOVQ DX, 64(SP) + MOVQ R9, 88(SP) + + // nextEmit := 0 + MOVQ DX, R10 + + // s := 1 + ADDQ $1, SI + + // nextHash := hash(load32(src, s), shift) + MOVL 0(SI), R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + +outer: + // for { etc } + + // skip := 32 + MOVQ $32, R12 + + // nextS := s + MOVQ SI, R13 + + // candidate := 0 + MOVQ $0, R15 + +inner0: + // for { etc } + + // s := nextS + MOVQ R13, SI + + // bytesBetweenHashLookups := skip >> 5 + MOVQ R12, R14 + SHRQ $5, R14 + + // nextS = s + bytesBetweenHashLookups + ADDQ R14, R13 + + // skip += bytesBetweenHashLookups + ADDQ R14, R12 + + // if nextS > sLimit { goto emitRemainder } + MOVQ R13, AX + SUBQ DX, AX + CMPQ AX, R9 + JA emitRemainder + + // candidate = int(table[nextHash]) + // XXX: MOVWQZX table-32768(SP)(R11*2), R15 + // XXX: 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 + BYTE $0x4e + BYTE $0x0f + BYTE $0xb7 + BYTE $0x7c + BYTE $0x5c + BYTE $0x78 + + // table[nextHash] = uint16(s) + MOVQ SI, AX + SUBQ DX, AX + + // XXX: MOVW AX, table-32768(SP)(R11*2) + // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) + BYTE $0x66 + BYTE $0x42 + BYTE $0x89 + BYTE $0x44 + BYTE $0x5c + BYTE $0x78 + + // nextHash = hash(load32(src, nextS), shift) + MOVL 0(R13), R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + + // if load32(src, s) != load32(src, candidate) { continue } break + MOVL 0(SI), AX + MOVL (DX)(R15*1), BX + CMPL AX, BX + JNE inner0 + +fourByteMatch: + // As per the encode_other.go code: + // + // A 4-byte match has been found. We'll later see etc. + + // !!! Jump to a fast path for short (<= 16 byte) literals. See the comment + // on inputMargin in encode.go. + MOVQ SI, AX + SUBQ R10, AX + CMPQ AX, $16 + JLE emitLiteralFastPath + + // ---------------------------------------- + // Begin inline of the emitLiteral call. + // + // d += emitLiteral(dst[d:], src[nextEmit:s]) + + MOVL AX, BX + SUBL $1, BX + + CMPL BX, $60 + JLT inlineEmitLiteralOneByte + CMPL BX, $256 + JLT inlineEmitLiteralTwoBytes + +inlineEmitLiteralThreeBytes: + MOVB $0xf4, 0(DI) + MOVW BX, 1(DI) + ADDQ $3, DI + JMP inlineEmitLiteralMemmove + +inlineEmitLiteralTwoBytes: + MOVB $0xf0, 0(DI) + MOVB BX, 1(DI) + ADDQ $2, DI + JMP inlineEmitLiteralMemmove + +inlineEmitLiteralOneByte: + SHLB $2, BX + MOVB BX, 0(DI) + ADDQ $1, DI + +inlineEmitLiteralMemmove: + // Spill local variables (registers) onto the stack; call; unspill. + // + // copy(dst[i:], lit) + // + // This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push + // DI, R10 and AX as arguments. + MOVQ DI, 0(SP) + MOVQ R10, 8(SP) + MOVQ AX, 16(SP) + ADDQ AX, DI // Finish the "d +=" part of "d += emitLiteral(etc)". + MOVQ SI, 72(SP) + MOVQ DI, 80(SP) + MOVQ R15, 112(SP) + CALL runtime·memmove(SB) + MOVQ 56(SP), CX + MOVQ 64(SP), DX + MOVQ 72(SP), SI + MOVQ 80(SP), DI + MOVQ 88(SP), R9 + MOVQ 112(SP), R15 + JMP inner1 + +inlineEmitLiteralEnd: + // End inline of the emitLiteral call. + // ---------------------------------------- + +emitLiteralFastPath: + // !!! Emit the 1-byte encoding "uint8(len(lit)-1)<<2". + MOVB AX, BX + SUBB $1, BX + SHLB $2, BX + MOVB BX, (DI) + ADDQ $1, DI + + // !!! Implement the copy from lit to dst as a 16-byte load and store. + // (Encode's documentation says that dst and src must not overlap.) + // + // This always copies 16 bytes, instead of only len(lit) bytes, but that's + // OK. Subsequent iterations will fix up the overrun. + // + // Note that on amd64, it is legal and cheap to issue unaligned 8-byte or + // 16-byte loads and stores. This technique probably wouldn't be as + // effective on architectures that are fussier about alignment. + MOVOU 0(R10), X0 + MOVOU X0, 0(DI) + ADDQ AX, DI + +inner1: + // for { etc } + + // base := s + MOVQ SI, R12 + + // !!! offset := base - candidate + MOVQ R12, R11 + SUBQ R15, R11 + SUBQ DX, R11 + + // ---------------------------------------- + // Begin inline of the extendMatch call. + // + // s = extendMatch(src, candidate+4, s+4) + + // !!! R14 = &src[len(src)] + MOVQ src_len+32(FP), R14 + ADDQ DX, R14 + + // !!! R13 = &src[len(src) - 8] + MOVQ R14, R13 + SUBQ $8, R13 + + // !!! R15 = &src[candidate + 4] + ADDQ $4, R15 + ADDQ DX, R15 + + // !!! s += 4 + ADDQ $4, SI + +inlineExtendMatchCmp8: + // As long as we are 8 or more bytes before the end of src, we can load and + // compare 8 bytes at a time. If those 8 bytes are equal, repeat. + CMPQ SI, R13 + JA inlineExtendMatchCmp1 + MOVQ (R15), AX + MOVQ (SI), BX + CMPQ AX, BX + JNE inlineExtendMatchBSF + ADDQ $8, R15 + ADDQ $8, SI + JMP inlineExtendMatchCmp8 + +inlineExtendMatchBSF: + // If those 8 bytes were not equal, XOR the two 8 byte values, and return + // the index of the first byte that differs. The BSF instruction finds the + // least significant 1 bit, the amd64 architecture is little-endian, and + // the shift by 3 converts a bit index to a byte index. + XORQ AX, BX + BSFQ BX, BX + SHRQ $3, BX + ADDQ BX, SI + JMP inlineExtendMatchEnd + +inlineExtendMatchCmp1: + // In src's tail, compare 1 byte at a time. + CMPQ SI, R14 + JAE inlineExtendMatchEnd + MOVB (R15), AX + MOVB (SI), BX + CMPB AX, BX + JNE inlineExtendMatchEnd + ADDQ $1, R15 + ADDQ $1, SI + JMP inlineExtendMatchCmp1 + +inlineExtendMatchEnd: + // End inline of the extendMatch call. + // ---------------------------------------- + + // ---------------------------------------- + // Begin inline of the emitCopy call. + // + // d += emitCopy(dst[d:], base-candidate, s-base) + + // !!! length := s - base + MOVQ SI, AX + SUBQ R12, AX + +inlineEmitCopyLoop0: + // for length >= 68 { etc } + CMPL AX, $68 + JLT inlineEmitCopyStep1 + + // Emit a length 64 copy, encoded as 3 bytes. + MOVB $0xfe, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + SUBL $64, AX + JMP inlineEmitCopyLoop0 + +inlineEmitCopyStep1: + // if length > 64 { etc } + CMPL AX, $64 + JLE inlineEmitCopyStep2 + + // Emit a length 60 copy, encoded as 3 bytes. + MOVB $0xee, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + SUBL $60, AX + +inlineEmitCopyStep2: + // if length >= 12 || offset >= 2048 { goto inlineEmitCopyStep3 } + CMPL AX, $12 + JGE inlineEmitCopyStep3 + CMPL R11, $2048 + JGE inlineEmitCopyStep3 + + // Emit the remaining copy, encoded as 2 bytes. + MOVB R11, 1(DI) + SHRL $8, R11 + SHLB $5, R11 + SUBB $4, AX + SHLB $2, AX + ORB AX, R11 + ORB $1, R11 + MOVB R11, 0(DI) + ADDQ $2, DI + JMP inlineEmitCopyEnd + +inlineEmitCopyStep3: + // Emit the remaining copy, encoded as 3 bytes. + SUBL $1, AX + SHLB $2, AX + ORB $2, AX + MOVB AX, 0(DI) + MOVW R11, 1(DI) + ADDQ $3, DI + +inlineEmitCopyEnd: + // End inline of the emitCopy call. + // ---------------------------------------- + + // nextEmit = s + MOVQ SI, R10 + + // if s >= sLimit { goto emitRemainder } + MOVQ SI, AX + SUBQ DX, AX + CMPQ AX, R9 + JAE emitRemainder + + // As per the encode_other.go code: + // + // We could immediately etc. + + // x := load64(src, s-1) + MOVQ -1(SI), R14 + + // prevHash := hash(uint32(x>>0), shift) + MOVL R14, R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + + // table[prevHash] = uint16(s-1) + MOVQ SI, AX + SUBQ DX, AX + SUBQ $1, AX + + // XXX: MOVW AX, table-32768(SP)(R11*2) + // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) + BYTE $0x66 + BYTE $0x42 + BYTE $0x89 + BYTE $0x44 + BYTE $0x5c + BYTE $0x78 + + // currHash := hash(uint32(x>>8), shift) + SHRQ $8, R14 + MOVL R14, R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + + // candidate = int(table[currHash]) + // XXX: MOVWQZX table-32768(SP)(R11*2), R15 + // XXX: 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 + BYTE $0x4e + BYTE $0x0f + BYTE $0xb7 + BYTE $0x7c + BYTE $0x5c + BYTE $0x78 + + // table[currHash] = uint16(s) + ADDQ $1, AX + + // XXX: MOVW AX, table-32768(SP)(R11*2) + // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) + BYTE $0x66 + BYTE $0x42 + BYTE $0x89 + BYTE $0x44 + BYTE $0x5c + BYTE $0x78 + + // if uint32(x>>8) == load32(src, candidate) { continue } + MOVL (DX)(R15*1), BX + CMPL R14, BX + JEQ inner1 + + // nextHash = hash(uint32(x>>16), shift) + SHRQ $8, R14 + MOVL R14, R11 + IMULL $0x1e35a7bd, R11 + SHRL CX, R11 + + // s++ + ADDQ $1, SI + + // break out of the inner1 for loop, i.e. continue the outer loop. + JMP outer + +emitRemainder: + // if nextEmit < len(src) { etc } + MOVQ src_len+32(FP), AX + ADDQ DX, AX + CMPQ R10, AX + JEQ encodeBlockEnd + + // d += emitLiteral(dst[d:], src[nextEmit:]) + // + // Push args. + MOVQ DI, 0(SP) + MOVQ $0, 8(SP) // Unnecessary, as the callee ignores it, but conservative. + MOVQ $0, 16(SP) // Unnecessary, as the callee ignores it, but conservative. + MOVQ R10, 24(SP) + SUBQ R10, AX + MOVQ AX, 32(SP) + MOVQ AX, 40(SP) // Unnecessary, as the callee ignores it, but conservative. + + // Spill local variables (registers) onto the stack; call; unspill. + MOVQ DI, 80(SP) + CALL ·emitLiteral(SB) + MOVQ 80(SP), DI + + // Finish the "d +=" part of "d += emitLiteral(etc)". + ADDQ 48(SP), DI + +encodeBlockEnd: + MOVQ dst_base+0(FP), AX + SUBQ AX, DI + MOVQ DI, d+48(FP) + RET diff --git a/vendor/github.com/golang/snappy/encode_other.go b/vendor/github.com/golang/snappy/encode_other.go new file mode 100644 index 00000000..dbcae905 --- /dev/null +++ b/vendor/github.com/golang/snappy/encode_other.go @@ -0,0 +1,238 @@ +// Copyright 2016 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !amd64 appengine !gc noasm + +package snappy + +func load32(b []byte, i int) uint32 { + b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. + return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 +} + +func load64(b []byte, i int) uint64 { + b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +// emitLiteral writes a literal chunk and returns the number of bytes written. +// +// It assumes that: +// dst is long enough to hold the encoded bytes +// 1 <= len(lit) && len(lit) <= 65536 +func emitLiteral(dst, lit []byte) int { + i, n := 0, uint(len(lit)-1) + switch { + case n < 60: + dst[0] = uint8(n)<<2 | tagLiteral + i = 1 + case n < 1<<8: + dst[0] = 60<<2 | tagLiteral + dst[1] = uint8(n) + i = 2 + default: + dst[0] = 61<<2 | tagLiteral + dst[1] = uint8(n) + dst[2] = uint8(n >> 8) + i = 3 + } + return i + copy(dst[i:], lit) +} + +// emitCopy writes a copy chunk and returns the number of bytes written. +// +// It assumes that: +// dst is long enough to hold the encoded bytes +// 1 <= offset && offset <= 65535 +// 4 <= length && length <= 65535 +func emitCopy(dst []byte, offset, length int) int { + i := 0 + // The maximum length for a single tagCopy1 or tagCopy2 op is 64 bytes. The + // threshold for this loop is a little higher (at 68 = 64 + 4), and the + // length emitted down below is is a little lower (at 60 = 64 - 4), because + // it's shorter to encode a length 67 copy as a length 60 tagCopy2 followed + // by a length 7 tagCopy1 (which encodes as 3+2 bytes) than to encode it as + // a length 64 tagCopy2 followed by a length 3 tagCopy2 (which encodes as + // 3+3 bytes). The magic 4 in the 64±4 is because the minimum length for a + // tagCopy1 op is 4 bytes, which is why a length 3 copy has to be an + // encodes-as-3-bytes tagCopy2 instead of an encodes-as-2-bytes tagCopy1. + for length >= 68 { + // Emit a length 64 copy, encoded as 3 bytes. + dst[i+0] = 63<<2 | tagCopy2 + dst[i+1] = uint8(offset) + dst[i+2] = uint8(offset >> 8) + i += 3 + length -= 64 + } + if length > 64 { + // Emit a length 60 copy, encoded as 3 bytes. + dst[i+0] = 59<<2 | tagCopy2 + dst[i+1] = uint8(offset) + dst[i+2] = uint8(offset >> 8) + i += 3 + length -= 60 + } + if length >= 12 || offset >= 2048 { + // Emit the remaining copy, encoded as 3 bytes. + dst[i+0] = uint8(length-1)<<2 | tagCopy2 + dst[i+1] = uint8(offset) + dst[i+2] = uint8(offset >> 8) + return i + 3 + } + // Emit the remaining copy, encoded as 2 bytes. + dst[i+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 + dst[i+1] = uint8(offset) + return i + 2 +} + +// extendMatch returns the largest k such that k <= len(src) and that +// src[i:i+k-j] and src[j:k] have the same contents. +// +// It assumes that: +// 0 <= i && i < j && j <= len(src) +func extendMatch(src []byte, i, j int) int { + for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { + } + return j +} + +func hash(u, shift uint32) uint32 { + return (u * 0x1e35a7bd) >> shift +} + +// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It +// assumes that the varint-encoded length of the decompressed bytes has already +// been written. +// +// It also assumes that: +// len(dst) >= MaxEncodedLen(len(src)) && +// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize +func encodeBlock(dst, src []byte) (d int) { + // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. + // The table element type is uint16, as s < sLimit and sLimit < len(src) + // and len(src) <= maxBlockSize and maxBlockSize == 65536. + const ( + maxTableSize = 1 << 14 + // tableMask is redundant, but helps the compiler eliminate bounds + // checks. + tableMask = maxTableSize - 1 + ) + shift := uint32(32 - 8) + for tableSize := 1 << 8; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 { + shift-- + } + // In Go, all array elements are zero-initialized, so there is no advantage + // to a smaller tableSize per se. However, it matches the C++ algorithm, + // and in the asm versions of this code, we can get away with zeroing only + // the first tableSize elements. + var table [maxTableSize]uint16 + + // sLimit is when to stop looking for offset/length copies. The inputMargin + // lets us use a fast path for emitLiteral in the main loop, while we are + // looking for copies. + sLimit := len(src) - inputMargin + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := 0 + + // The encoded form must start with a literal, as there are no previous + // bytes to copy, so we start looking for hash matches at s == 1. + s := 1 + nextHash := hash(load32(src, s), shift) + + for { + // Copied from the C++ snappy implementation: + // + // Heuristic match skipping: If 32 bytes are scanned with no matches + // found, start looking only at every other byte. If 32 more bytes are + // scanned (or skipped), look at every third byte, etc.. When a match + // is found, immediately go back to looking at every byte. This is a + // small loss (~5% performance, ~0.1% density) for compressible data + // due to more bookkeeping, but for non-compressible data (such as + // JPEG) it's a huge win since the compressor quickly "realizes" the + // data is incompressible and doesn't bother looking for matches + // everywhere. + // + // The "skip" variable keeps track of how many bytes there are since + // the last match; dividing it by 32 (ie. right-shifting by five) gives + // the number of bytes to move ahead for each iteration. + skip := 32 + + nextS := s + candidate := 0 + for { + s = nextS + bytesBetweenHashLookups := skip >> 5 + nextS = s + bytesBetweenHashLookups + skip += bytesBetweenHashLookups + if nextS > sLimit { + goto emitRemainder + } + candidate = int(table[nextHash&tableMask]) + table[nextHash&tableMask] = uint16(s) + nextHash = hash(load32(src, nextS), shift) + if load32(src, s) == load32(src, candidate) { + break + } + } + + // A 4-byte match has been found. We'll later see if more than 4 bytes + // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit + // them as literal bytes. + d += emitLiteral(dst[d:], src[nextEmit:s]) + + // Call emitCopy, and then see if another emitCopy could be our next + // move. Repeat until we find no match for the input immediately after + // what was consumed by the last emitCopy call. + // + // If we exit this loop normally then we need to call emitLiteral next, + // though we don't yet know how big the literal will be. We handle that + // by proceeding to the next iteration of the main loop. We also can + // exit this loop via goto if we get close to exhausting the input. + for { + // Invariant: we have a 4-byte match at s, and no need to emit any + // literal bytes prior to s. + base := s + + // Extend the 4-byte match as long as possible. + // + // This is an inlined version of: + // s = extendMatch(src, candidate+4, s+4) + s += 4 + for i := candidate + 4; s < len(src) && src[i] == src[s]; i, s = i+1, s+1 { + } + + d += emitCopy(dst[d:], base-candidate, s-base) + nextEmit = s + if s >= sLimit { + goto emitRemainder + } + + // We could immediately start working at s now, but to improve + // compression we first update the hash table at s-1 and at s. If + // another emitCopy is not our next move, also calculate nextHash + // at s+1. At least on GOARCH=amd64, these three hash calculations + // are faster as one load64 call (with some shifts) instead of + // three load32 calls. + x := load64(src, s-1) + prevHash := hash(uint32(x>>0), shift) + table[prevHash&tableMask] = uint16(s - 1) + currHash := hash(uint32(x>>8), shift) + candidate = int(table[currHash&tableMask]) + table[currHash&tableMask] = uint16(s) + if uint32(x>>8) != load32(src, candidate) { + nextHash = hash(uint32(x>>16), shift) + s++ + break + } + } + } + +emitRemainder: + if nextEmit < len(src) { + d += emitLiteral(dst[d:], src[nextEmit:]) + } + return d +} diff --git a/vendor/github.com/golang/snappy/snappy.go b/vendor/github.com/golang/snappy/snappy.go new file mode 100644 index 00000000..0cf5e379 --- /dev/null +++ b/vendor/github.com/golang/snappy/snappy.go @@ -0,0 +1,87 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package snappy implements the snappy block-based compression format. +// It aims for very high speeds and reasonable compression. +// +// The C++ snappy implementation is at https://github.com/google/snappy +package snappy // import "github.com/golang/snappy" + +import ( + "hash/crc32" +) + +/* +Each encoded block begins with the varint-encoded length of the decoded data, +followed by a sequence of chunks. Chunks begin and end on byte boundaries. The +first byte of each chunk is broken into its 2 least and 6 most significant bits +called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. +Zero means a literal tag. All other values mean a copy tag. + +For literal tags: + - If m < 60, the next 1 + m bytes are literal bytes. + - Otherwise, let n be the little-endian unsigned integer denoted by the next + m - 59 bytes. The next 1 + n bytes after that are literal bytes. + +For copy tags, length bytes are copied from offset bytes ago, in the style of +Lempel-Ziv compression algorithms. In particular: + - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). + The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 + of the offset. The next byte is bits 0-7 of the offset. + - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). + The length is 1 + m. The offset is the little-endian unsigned integer + denoted by the next 2 bytes. + - For l == 3, this tag is a legacy format that is no longer issued by most + encoders. Nonetheless, the offset ranges in [0, 1<<32) and the length in + [1, 65). The length is 1 + m. The offset is the little-endian unsigned + integer denoted by the next 4 bytes. +*/ +const ( + tagLiteral = 0x00 + tagCopy1 = 0x01 + tagCopy2 = 0x02 + tagCopy4 = 0x03 +) + +const ( + checksumSize = 4 + chunkHeaderSize = 4 + magicChunk = "\xff\x06\x00\x00" + magicBody + magicBody = "sNaPpY" + + // maxBlockSize is the maximum size of the input to encodeBlock. It is not + // part of the wire format per se, but some parts of the encoder assume + // that an offset fits into a uint16. + // + // Also, for the framing format (Writer type instead of Encode function), + // https://github.com/google/snappy/blob/master/framing_format.txt says + // that "the uncompressed data in a chunk must be no longer than 65536 + // bytes". + maxBlockSize = 65536 + + // maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is + // hard coded to be a const instead of a variable, so that obufLen can also + // be a const. Their equivalence is confirmed by + // TestMaxEncodedLenOfMaxBlockSize. + maxEncodedLenOfMaxBlockSize = 76490 + + obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize + obufLen = obufHeaderLen + maxEncodedLenOfMaxBlockSize +) + +const ( + chunkTypeCompressedData = 0x00 + chunkTypeUncompressedData = 0x01 + chunkTypePadding = 0xfe + chunkTypeStreamIdentifier = 0xff +) + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// crc implements the checksum specified in section 3 of +// https://github.com/google/snappy/blob/master/framing_format.txt +func crc(b []byte) uint32 { + c := crc32.Update(0, crcTable, b) + return uint32(c>>15|c<<17) + 0xa282ead8 +} diff --git a/vendor/github.com/gorilla/websocket/README.md b/vendor/github.com/gorilla/websocket/README.md index 33c3d2be..20e391f8 100644 --- a/vendor/github.com/gorilla/websocket/README.md +++ b/vendor/github.com/gorilla/websocket/README.md @@ -51,7 +51,7 @@ subdirectory](https://github.com/gorilla/websocket/tree/master/examples/autobahn Write message using io.WriteCloserYesNo, see note 3 -Notes: +Notes: 1. Large messages are fragmented in [Chrome's new WebSocket implementation](http://www.ietf.org/mail-archive/web/hybi/current/msg10503.html). 2. The application can get the type of a received data message by implementing diff --git a/vendor/github.com/gorilla/websocket/client.go b/vendor/github.com/gorilla/websocket/client.go index 43a87c75..41f8ed53 100644 --- a/vendor/github.com/gorilla/websocket/client.go +++ b/vendor/github.com/gorilla/websocket/client.go @@ -5,10 +5,8 @@ package websocket import ( - "bufio" "bytes" "crypto/tls" - "encoding/base64" "errors" "io" "io/ioutil" @@ -88,50 +86,6 @@ type Dialer struct { var errMalformedURL = errors.New("malformed ws or wss URL") -// parseURL parses the URL. -// -// This function is a replacement for the standard library url.Parse function. -// In Go 1.4 and earlier, url.Parse loses information from the path. -func parseURL(s string) (*url.URL, error) { - // From the RFC: - // - // ws-URI = "ws:" "//" host [ ":" port ] path [ "?" query ] - // wss-URI = "wss:" "//" host [ ":" port ] path [ "?" query ] - var u url.URL - switch { - case strings.HasPrefix(s, "ws://"): - u.Scheme = "ws" - s = s[len("ws://"):] - case strings.HasPrefix(s, "wss://"): - u.Scheme = "wss" - s = s[len("wss://"):] - default: - return nil, errMalformedURL - } - - if i := strings.Index(s, "?"); i >= 0 { - u.RawQuery = s[i+1:] - s = s[:i] - } - - if i := strings.Index(s, "/"); i >= 0 { - u.Opaque = s[i:] - s = s[:i] - } else { - u.Opaque = "/" - } - - u.Host = s - - if strings.Contains(u.Host, "@") { - // Don't bother parsing user information because user information is - // not allowed in websocket URIs. - return nil, errMalformedURL - } - - return &u, nil -} - func hostPortNoPort(u *url.URL) (hostPort, hostNoPort string) { hostPort = u.Host hostNoPort = u.Host @@ -150,11 +104,15 @@ func hostPortNoPort(u *url.URL) (hostPort, hostNoPort string) { return hostPort, hostNoPort } -// DefaultDialer is a dialer with all fields set to the default zero values. +// DefaultDialer is a dialer with all fields set to the default values. var DefaultDialer = &Dialer{ - Proxy: http.ProxyFromEnvironment, + Proxy: http.ProxyFromEnvironment, + HandshakeTimeout: 45 * time.Second, } +// nilDialer is dialer to use when receiver is nil. +var nilDialer Dialer = *DefaultDialer + // Dial creates a new client connection. Use requestHeader to specify the // origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies (Cookie). // Use the response.Header to get the selected subprotocol @@ -167,9 +125,7 @@ var DefaultDialer = &Dialer{ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Response, error) { if d == nil { - d = &Dialer{ - Proxy: http.ProxyFromEnvironment, - } + d = &nilDialer } challengeKey, err := generateChallengeKey() @@ -177,7 +133,7 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re return nil, nil, err } - u, err := parseURL(urlStr) + u, err := url.Parse(urlStr) if err != nil { return nil, nil, err } @@ -237,31 +193,15 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re k == "Sec-Websocket-Extensions" || (k == "Sec-Websocket-Protocol" && len(d.Subprotocols) > 0): return nil, nil, errors.New("websocket: duplicate header not allowed: " + k) + case k == "Sec-Websocket-Protocol": + req.Header["Sec-WebSocket-Protocol"] = vs default: req.Header[k] = vs } } if d.EnableCompression { - req.Header.Set("Sec-Websocket-Extensions", "permessage-deflate; server_no_context_takeover; client_no_context_takeover") - } - - hostPort, hostNoPort := hostPortNoPort(u) - - var proxyURL *url.URL - // Check wether the proxy method has been configured - if d.Proxy != nil { - proxyURL, err = d.Proxy(req) - } - if err != nil { - return nil, nil, err - } - - var targetHostPort string - if proxyURL != nil { - targetHostPort, _ = hostPortNoPort(proxyURL) - } else { - targetHostPort = hostPort + req.Header["Sec-WebSocket-Extensions"] = []string{"permessage-deflate; server_no_context_takeover; client_no_context_takeover"} } var deadline time.Time @@ -269,13 +209,47 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re deadline = time.Now().Add(d.HandshakeTimeout) } + // Get network dial function. netDial := d.NetDial if netDial == nil { netDialer := &net.Dialer{Deadline: deadline} netDial = netDialer.Dial } - netConn, err := netDial("tcp", targetHostPort) + // If needed, wrap the dial function to set the connection deadline. + if !deadline.Equal(time.Time{}) { + forwardDial := netDial + netDial = func(network, addr string) (net.Conn, error) { + c, err := forwardDial(network, addr) + if err != nil { + return nil, err + } + err = c.SetDeadline(deadline) + if err != nil { + c.Close() + return nil, err + } + return c, nil + } + } + + // If needed, wrap the dial function to connect through a proxy. + if d.Proxy != nil { + proxyURL, err := d.Proxy(req) + if err != nil { + return nil, nil, err + } + if proxyURL != nil { + dialer, err := proxy_FromURL(proxyURL, netDialerFunc(netDial)) + if err != nil { + return nil, nil, err + } + netDial = dialer.Dial + } + } + + hostPort, hostNoPort := hostPortNoPort(u) + netConn, err := netDial("tcp", hostPort) if err != nil { return nil, nil, err } @@ -286,42 +260,6 @@ func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Re } }() - if err := netConn.SetDeadline(deadline); err != nil { - return nil, nil, err - } - - if proxyURL != nil { - connectHeader := make(http.Header) - if user := proxyURL.User; user != nil { - proxyUser := user.Username() - if proxyPassword, passwordSet := user.Password(); passwordSet { - credential := base64.StdEncoding.EncodeToString([]byte(proxyUser + ":" + proxyPassword)) - connectHeader.Set("Proxy-Authorization", "Basic "+credential) - } - } - connectReq := &http.Request{ - Method: "CONNECT", - URL: &url.URL{Opaque: hostPort}, - Host: hostPort, - Header: connectHeader, - } - - connectReq.Write(netConn) - - // Read response. - // Okay to use and discard buffered reader here, because - // TLS server will not speak until spoken to. - br := bufio.NewReader(netConn) - resp, err := http.ReadResponse(br, connectReq) - if err != nil { - return nil, nil, err - } - if resp.StatusCode != 200 { - f := strings.SplitN(resp.Status, " ", 2) - return nil, nil, errors.New(f[1]) - } - } - if u.Scheme == "https" { cfg := cloneTLSConfig(d.TLSClientConfig) if cfg.ServerName == "" { diff --git a/vendor/github.com/gorilla/websocket/conn.go b/vendor/github.com/gorilla/websocket/conn.go index 97e1dbac..5f46bf4a 100644 --- a/vendor/github.com/gorilla/websocket/conn.go +++ b/vendor/github.com/gorilla/websocket/conn.go @@ -76,7 +76,7 @@ const ( // is UTF-8 encoded text. PingMessage = 9 - // PongMessage denotes a ping control message. The optional message payload + // PongMessage denotes a pong control message. The optional message payload // is UTF-8 encoded text. PongMessage = 10 ) @@ -100,9 +100,8 @@ func (e *netError) Error() string { return e.msg } func (e *netError) Temporary() bool { return e.temporary } func (e *netError) Timeout() bool { return e.timeout } -// CloseError represents close frame. +// CloseError represents a close message. type CloseError struct { - // Code is defined in RFC 6455, section 11.7. Code int @@ -343,7 +342,8 @@ func (c *Conn) Subprotocol() string { return c.subprotocol } -// Close closes the underlying network connection without sending or waiting for a close frame. +// Close closes the underlying network connection without sending or waiting +// for a close message. func (c *Conn) Close() error { return c.conn.Close() } @@ -370,7 +370,7 @@ func (c *Conn) writeFatal(err error) error { return err } -func (c *Conn) write(frameType int, deadline time.Time, bufs ...[]byte) error { +func (c *Conn) write(frameType int, deadline time.Time, buf0, buf1 []byte) error { <-c.mu defer func() { c.mu <- true }() @@ -382,15 +382,14 @@ func (c *Conn) write(frameType int, deadline time.Time, bufs ...[]byte) error { } c.conn.SetWriteDeadline(deadline) - for _, buf := range bufs { - if len(buf) > 0 { - _, err := c.conn.Write(buf) - if err != nil { - return c.writeFatal(err) - } - } + if len(buf1) == 0 { + _, err = c.conn.Write(buf0) + } else { + err = c.writeBufs(buf0, buf1) + } + if err != nil { + return c.writeFatal(err) } - if frameType == CloseMessage { c.writeFatal(ErrCloseSent) } @@ -484,6 +483,9 @@ func (c *Conn) prepWrite(messageType int) error { // // There can be at most one open writer on a connection. NextWriter closes the // previous writer if the application has not already done so. +// +// All message types (TextMessage, BinaryMessage, CloseMessage, PingMessage and +// PongMessage) are supported. func (c *Conn) NextWriter(messageType int) (io.WriteCloser, error) { if err := c.prepWrite(messageType); err != nil { return nil, err @@ -764,7 +766,6 @@ func (c *Conn) SetWriteDeadline(t time.Time) error { // Read methods func (c *Conn) advanceFrame() (int, error) { - // 1. Skip remainder of previous frame. if c.readRemaining > 0 { @@ -1033,7 +1034,7 @@ func (c *Conn) SetReadDeadline(t time.Time) error { } // SetReadLimit sets the maximum size for a message read from the peer. If a -// message exceeds the limit, the connection sends a close frame to the peer +// message exceeds the limit, the connection sends a close message to the peer // and returns ErrReadLimit to the application. func (c *Conn) SetReadLimit(limit int64) { c.readLimit = limit @@ -1046,24 +1047,22 @@ func (c *Conn) CloseHandler() func(code int, text string) error { // SetCloseHandler sets the handler for close messages received from the peer. // The code argument to h is the received close code or CloseNoStatusReceived -// if the close message is empty. The default close handler sends a close frame -// back to the peer. +// if the close message is empty. The default close handler sends a close +// message back to the peer. // -// The application must read the connection to process close messages as -// described in the section on Control Frames above. +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// close messages as described in the section on Control Messages above. // -// The connection read methods return a CloseError when a close frame is +// The connection read methods return a CloseError when a close message is // received. Most applications should handle close messages as part of their // normal error handling. Applications should only set a close handler when the -// application must perform some action before sending a close frame back to +// application must perform some action before sending a close message back to // the peer. func (c *Conn) SetCloseHandler(h func(code int, text string) error) { if h == nil { h = func(code int, text string) error { - message := []byte{} - if code != CloseNoStatusReceived { - message = FormatCloseMessage(code, "") - } + message := FormatCloseMessage(code, "") c.WriteControl(CloseMessage, message, time.Now().Add(writeWait)) return nil } @@ -1077,11 +1076,12 @@ func (c *Conn) PingHandler() func(appData string) error { } // SetPingHandler sets the handler for ping messages received from the peer. -// The appData argument to h is the PING frame application data. The default +// The appData argument to h is the PING message application data. The default // ping handler sends a pong to the peer. // -// The application must read the connection to process ping messages as -// described in the section on Control Frames above. +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// ping messages as described in the section on Control Messages above. func (c *Conn) SetPingHandler(h func(appData string) error) { if h == nil { h = func(message string) error { @@ -1103,11 +1103,12 @@ func (c *Conn) PongHandler() func(appData string) error { } // SetPongHandler sets the handler for pong messages received from the peer. -// The appData argument to h is the PONG frame application data. The default +// The appData argument to h is the PONG message application data. The default // pong handler does nothing. // -// The application must read the connection to process ping messages as -// described in the section on Control Frames above. +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// pong messages as described in the section on Control Messages above. func (c *Conn) SetPongHandler(h func(appData string) error) { if h == nil { h = func(string) error { return nil } @@ -1141,7 +1142,14 @@ func (c *Conn) SetCompressionLevel(level int) error { } // FormatCloseMessage formats closeCode and text as a WebSocket close message. +// An empty message is returned for code CloseNoStatusReceived. func FormatCloseMessage(closeCode int, text string) []byte { + if closeCode == CloseNoStatusReceived { + // Return empty message because it's illegal to send + // CloseNoStatusReceived. Return non-nil value in case application + // checks for nil. + return []byte{} + } buf := make([]byte, 2+len(text)) binary.BigEndian.PutUint16(buf, uint16(closeCode)) copy(buf[2:], text) diff --git a/vendor/github.com/gorilla/websocket/conn_write.go b/vendor/github.com/gorilla/websocket/conn_write.go new file mode 100644 index 00000000..a509a21f --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_write.go @@ -0,0 +1,15 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.8 + +package websocket + +import "net" + +func (c *Conn) writeBufs(bufs ...[]byte) error { + b := net.Buffers(bufs) + _, err := b.WriteTo(c.conn) + return err +} diff --git a/vendor/github.com/gorilla/websocket/conn_write_legacy.go b/vendor/github.com/gorilla/websocket/conn_write_legacy.go new file mode 100644 index 00000000..37edaff5 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_write_legacy.go @@ -0,0 +1,18 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.8 + +package websocket + +func (c *Conn) writeBufs(bufs ...[]byte) error { + for _, buf := range bufs { + if len(buf) > 0 { + if _, err := c.conn.Write(buf); err != nil { + return err + } + } + } + return nil +} diff --git a/vendor/github.com/gorilla/websocket/doc.go b/vendor/github.com/gorilla/websocket/doc.go index e291a952..dcce1a63 100644 --- a/vendor/github.com/gorilla/websocket/doc.go +++ b/vendor/github.com/gorilla/websocket/doc.go @@ -6,9 +6,8 @@ // // Overview // -// The Conn type represents a WebSocket connection. A server application uses -// the Upgrade function from an Upgrader object with a HTTP request handler -// to get a pointer to a Conn: +// The Conn type represents a WebSocket connection. A server application calls +// the Upgrader.Upgrade method from an HTTP request handler to get a *Conn: // // var upgrader = websocket.Upgrader{ // ReadBufferSize: 1024, @@ -31,10 +30,12 @@ // for { // messageType, p, err := conn.ReadMessage() // if err != nil { +// log.Println(err) // return // } -// if err = conn.WriteMessage(messageType, p); err != nil { -// return err +// if err := conn.WriteMessage(messageType, p); err != nil { +// log.Println(err) +// return // } // } // @@ -85,20 +86,26 @@ // and pong. Call the connection WriteControl, WriteMessage or NextWriter // methods to send a control message to the peer. // -// Connections handle received close messages by sending a close message to the -// peer and returning a *CloseError from the the NextReader, ReadMessage or the -// message Read method. +// Connections handle received close messages by calling the handler function +// set with the SetCloseHandler method and by returning a *CloseError from the +// NextReader, ReadMessage or the message Read method. The default close +// handler sends a close message to the peer. // -// Connections handle received ping and pong messages by invoking callback -// functions set with SetPingHandler and SetPongHandler methods. The callback -// functions are called from the NextReader, ReadMessage and the message Read -// methods. +// Connections handle received ping messages by calling the handler function +// set with the SetPingHandler method. The default ping handler sends a pong +// message to the peer. // -// The default ping handler sends a pong to the peer. The application's reading -// goroutine can block for a short time while the handler writes the pong data -// to the connection. +// Connections handle received pong messages by calling the handler function +// set with the SetPongHandler method. The default pong handler does nothing. +// If an application sends ping messages, then the application should set a +// pong handler to receive the corresponding pong. // -// The application must read the connection to process ping, pong and close +// The control message handler functions are called from the NextReader, +// ReadMessage and message reader Read methods. The default close and ping +// handlers can block these methods for a short time when the handler writes to +// the connection. +// +// The application must read the connection to process close, ping and pong // messages sent from the peer. If the application is not otherwise interested // in messages from the peer, then the application should start a goroutine to // read and discard messages from the peer. A simple example is: @@ -137,19 +144,12 @@ // method fails the WebSocket handshake with HTTP status 403. // // If the CheckOrigin field is nil, then the Upgrader uses a safe default: fail -// the handshake if the Origin request header is present and not equal to the -// Host request header. +// the handshake if the Origin request header is present and the Origin host is +// not equal to the Host request header. // -// An application can allow connections from any origin by specifying a -// function that always returns true: -// -// var upgrader = websocket.Upgrader{ -// CheckOrigin: func(r *http.Request) bool { return true }, -// } -// -// The deprecated Upgrade function does not enforce an origin policy. It's the -// application's responsibility to check the Origin header before calling -// Upgrade. +// The deprecated package-level Upgrade function does not perform origin +// checking. The application is responsible for checking the Origin header +// before calling the Upgrade function. // // Compression EXPERIMENTAL // diff --git a/vendor/github.com/gorilla/websocket/json.go b/vendor/github.com/gorilla/websocket/json.go index 4f0e3687..dc2c1f64 100644 --- a/vendor/github.com/gorilla/websocket/json.go +++ b/vendor/github.com/gorilla/websocket/json.go @@ -9,12 +9,14 @@ import ( "io" ) -// WriteJSON is deprecated, use c.WriteJSON instead. +// WriteJSON writes the JSON encoding of v as a message. +// +// Deprecated: Use c.WriteJSON instead. func WriteJSON(c *Conn, v interface{}) error { return c.WriteJSON(v) } -// WriteJSON writes the JSON encoding of v to the connection. +// WriteJSON writes the JSON encoding of v as a message. // // See the documentation for encoding/json Marshal for details about the // conversion of Go values to JSON. @@ -31,7 +33,10 @@ func (c *Conn) WriteJSON(v interface{}) error { return err2 } -// ReadJSON is deprecated, use c.ReadJSON instead. +// ReadJSON reads the next JSON-encoded message from the connection and stores +// it in the value pointed to by v. +// +// Deprecated: Use c.ReadJSON instead. func ReadJSON(c *Conn, v interface{}) error { return c.ReadJSON(v) } diff --git a/vendor/github.com/gorilla/websocket/mask.go b/vendor/github.com/gorilla/websocket/mask.go index 6a88bbc7..577fce9e 100644 --- a/vendor/github.com/gorilla/websocket/mask.go +++ b/vendor/github.com/gorilla/websocket/mask.go @@ -11,7 +11,6 @@ import "unsafe" const wordSize = int(unsafe.Sizeof(uintptr(0))) func maskBytes(key [4]byte, pos int, b []byte) int { - // Mask one byte at a time for small buffers. if len(b) < 2*wordSize { for i := range b { diff --git a/vendor/github.com/gorilla/websocket/proxy.go b/vendor/github.com/gorilla/websocket/proxy.go new file mode 100644 index 00000000..bf2478e4 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/proxy.go @@ -0,0 +1,77 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "encoding/base64" + "errors" + "net" + "net/http" + "net/url" + "strings" +) + +type netDialerFunc func(network, addr string) (net.Conn, error) + +func (fn netDialerFunc) Dial(network, addr string) (net.Conn, error) { + return fn(network, addr) +} + +func init() { + proxy_RegisterDialerType("http", func(proxyURL *url.URL, forwardDialer proxy_Dialer) (proxy_Dialer, error) { + return &httpProxyDialer{proxyURL: proxyURL, fowardDial: forwardDialer.Dial}, nil + }) +} + +type httpProxyDialer struct { + proxyURL *url.URL + fowardDial func(network, addr string) (net.Conn, error) +} + +func (hpd *httpProxyDialer) Dial(network string, addr string) (net.Conn, error) { + hostPort, _ := hostPortNoPort(hpd.proxyURL) + conn, err := hpd.fowardDial(network, hostPort) + if err != nil { + return nil, err + } + + connectHeader := make(http.Header) + if user := hpd.proxyURL.User; user != nil { + proxyUser := user.Username() + if proxyPassword, passwordSet := user.Password(); passwordSet { + credential := base64.StdEncoding.EncodeToString([]byte(proxyUser + ":" + proxyPassword)) + connectHeader.Set("Proxy-Authorization", "Basic "+credential) + } + } + + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: addr}, + Host: addr, + Header: connectHeader, + } + + if err := connectReq.Write(conn); err != nil { + conn.Close() + return nil, err + } + + // Read response. It's OK to use and discard buffered reader here becaue + // the remote server does not speak until spoken to. + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, connectReq) + if err != nil { + conn.Close() + return nil, err + } + + if resp.StatusCode != 200 { + conn.Close() + f := strings.SplitN(resp.Status, " ", 2) + return nil, errors.New(f[1]) + } + return conn, nil +} diff --git a/vendor/github.com/gorilla/websocket/server.go b/vendor/github.com/gorilla/websocket/server.go index 3495e0f1..aee27052 100644 --- a/vendor/github.com/gorilla/websocket/server.go +++ b/vendor/github.com/gorilla/websocket/server.go @@ -44,8 +44,12 @@ type Upgrader struct { Error func(w http.ResponseWriter, r *http.Request, status int, reason error) // CheckOrigin returns true if the request Origin header is acceptable. If - // CheckOrigin is nil, the host in the Origin header must not be set or - // must match the host of the request. + // CheckOrigin is nil, then a safe default is used: return false if the + // Origin request header is present and the origin host is not equal to + // request Host header. + // + // A CheckOrigin function should carefully validate the request origin to + // prevent cross-site request forgery. CheckOrigin func(r *http.Request) bool // EnableCompression specify if the server should attempt to negotiate per @@ -76,7 +80,7 @@ func checkSameOrigin(r *http.Request) bool { if err != nil { return false } - return u.Host == r.Host + return equalASCIIFold(u.Host, r.Host) } func (u *Upgrader) selectSubprotocol(r *http.Request, responseHeader http.Header) string { @@ -99,42 +103,44 @@ func (u *Upgrader) selectSubprotocol(r *http.Request, responseHeader http.Header // // The responseHeader is included in the response to the client's upgrade // request. Use the responseHeader to specify cookies (Set-Cookie) and the -// application negotiated subprotocol (Sec-Websocket-Protocol). +// application negotiated subprotocol (Sec-WebSocket-Protocol). // // If the upgrade fails, then Upgrade replies to the client with an HTTP error // response. func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*Conn, error) { - if r.Method != "GET" { - return u.returnError(w, r, http.StatusMethodNotAllowed, "websocket: not a websocket handshake: request method is not GET") - } - - if _, ok := responseHeader["Sec-Websocket-Extensions"]; ok { - return u.returnError(w, r, http.StatusInternalServerError, "websocket: application specific 'Sec-Websocket-Extensions' headers are unsupported") - } + const badHandshake = "websocket: the client is not using the websocket protocol: " if !tokenListContainsValue(r.Header, "Connection", "upgrade") { - return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: 'upgrade' token not found in 'Connection' header") + return u.returnError(w, r, http.StatusBadRequest, badHandshake+"'upgrade' token not found in 'Connection' header") } if !tokenListContainsValue(r.Header, "Upgrade", "websocket") { - return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: 'websocket' token not found in 'Upgrade' header") + return u.returnError(w, r, http.StatusBadRequest, badHandshake+"'websocket' token not found in 'Upgrade' header") + } + + if r.Method != "GET" { + return u.returnError(w, r, http.StatusMethodNotAllowed, badHandshake+"request method is not GET") } if !tokenListContainsValue(r.Header, "Sec-Websocket-Version", "13") { return u.returnError(w, r, http.StatusBadRequest, "websocket: unsupported version: 13 not found in 'Sec-Websocket-Version' header") } + if _, ok := responseHeader["Sec-Websocket-Extensions"]; ok { + return u.returnError(w, r, http.StatusInternalServerError, "websocket: application specific 'Sec-WebSocket-Extensions' headers are unsupported") + } + checkOrigin := u.CheckOrigin if checkOrigin == nil { checkOrigin = checkSameOrigin } if !checkOrigin(r) { - return u.returnError(w, r, http.StatusForbidden, "websocket: 'Origin' header value not allowed") + return u.returnError(w, r, http.StatusForbidden, "websocket: request origin not allowed by Upgrader.CheckOrigin") } challengeKey := r.Header.Get("Sec-Websocket-Key") if challengeKey == "" { - return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: `Sec-Websocket-Key' header is missing or blank") + return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: `Sec-WebSocket-Key' header is missing or blank") } subprotocol := u.selectSubprotocol(r, responseHeader) @@ -184,12 +190,12 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade p = append(p, computeAcceptKey(challengeKey)...) p = append(p, "\r\n"...) if c.subprotocol != "" { - p = append(p, "Sec-Websocket-Protocol: "...) + p = append(p, "Sec-WebSocket-Protocol: "...) p = append(p, c.subprotocol...) p = append(p, "\r\n"...) } if compress { - p = append(p, "Sec-Websocket-Extensions: permessage-deflate; server_no_context_takeover; client_no_context_takeover\r\n"...) + p = append(p, "Sec-WebSocket-Extensions: permessage-deflate; server_no_context_takeover; client_no_context_takeover\r\n"...) } for k, vs := range responseHeader { if k == "Sec-Websocket-Protocol" { @@ -230,13 +236,14 @@ func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeade // Upgrade upgrades the HTTP server connection to the WebSocket protocol. // -// This function is deprecated, use websocket.Upgrader instead. +// Deprecated: Use websocket.Upgrader instead. // -// The application is responsible for checking the request origin before -// calling Upgrade. An example implementation of the same origin policy is: +// Upgrade does not perform origin checking. The application is responsible for +// checking the Origin header before calling Upgrade. An example implementation +// of the same origin policy check is: // // if req.Header.Get("Origin") != "http://"+req.Host { -// http.Error(w, "Origin not allowed", 403) +// http.Error(w, "Origin not allowed", http.StatusForbidden) // return // } // diff --git a/vendor/github.com/gorilla/websocket/util.go b/vendor/github.com/gorilla/websocket/util.go index 9a4908df..385fa01b 100644 --- a/vendor/github.com/gorilla/websocket/util.go +++ b/vendor/github.com/gorilla/websocket/util.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "strings" + "unicode/utf8" ) var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") @@ -111,14 +112,14 @@ func nextTokenOrQuoted(s string) (value string, rest string) { case escape: escape = false p[j] = b - j += 1 + j++ case b == '\\': escape = true case b == '"': return string(p[:j]), s[i+1:] default: p[j] = b - j += 1 + j++ } } return "", "" @@ -127,8 +128,31 @@ func nextTokenOrQuoted(s string) (value string, rest string) { return "", "" } +// equalASCIIFold returns true if s is equal to t with ASCII case folding. +func equalASCIIFold(s, t string) bool { + for s != "" && t != "" { + sr, size := utf8.DecodeRuneInString(s) + s = s[size:] + tr, size := utf8.DecodeRuneInString(t) + t = t[size:] + if sr == tr { + continue + } + if 'A' <= sr && sr <= 'Z' { + sr = sr + 'a' - 'A' + } + if 'A' <= tr && tr <= 'Z' { + tr = tr + 'a' - 'A' + } + if sr != tr { + return false + } + } + return s == t +} + // tokenListContainsValue returns true if the 1#token header with the given -// name contains token. +// name contains a token equal to value with ASCII case folding. func tokenListContainsValue(header http.Header, name string, value string) bool { headers: for _, s := range header[name] { @@ -142,7 +166,7 @@ headers: if s != "" && s[0] != ',' { continue headers } - if strings.EqualFold(t, value) { + if equalASCIIFold(t, value) { return true } if s == "" { @@ -156,7 +180,6 @@ headers: // parseExtensiosn parses WebSocket extensions from a header. func parseExtensions(header http.Header) []map[string]string { - // From RFC 6455: // // Sec-WebSocket-Extensions = extension-list diff --git a/vendor/github.com/gorilla/websocket/x_net_proxy.go b/vendor/github.com/gorilla/websocket/x_net_proxy.go new file mode 100644 index 00000000..2e668f6b --- /dev/null +++ b/vendor/github.com/gorilla/websocket/x_net_proxy.go @@ -0,0 +1,473 @@ +// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. +//go:generate bundle -o x_net_proxy.go golang.org/x/net/proxy + +// Package proxy provides support for a variety of protocols to proxy network +// data. +// + +package websocket + +import ( + "errors" + "io" + "net" + "net/url" + "os" + "strconv" + "strings" + "sync" +) + +type proxy_direct struct{} + +// Direct is a direct proxy: one that makes network connections directly. +var proxy_Direct = proxy_direct{} + +func (proxy_direct) Dial(network, addr string) (net.Conn, error) { + return net.Dial(network, addr) +} + +// A PerHost directs connections to a default Dialer unless the host name +// requested matches one of a number of exceptions. +type proxy_PerHost struct { + def, bypass proxy_Dialer + + bypassNetworks []*net.IPNet + bypassIPs []net.IP + bypassZones []string + bypassHosts []string +} + +// NewPerHost returns a PerHost Dialer that directs connections to either +// defaultDialer or bypass, depending on whether the connection matches one of +// the configured rules. +func proxy_NewPerHost(defaultDialer, bypass proxy_Dialer) *proxy_PerHost { + return &proxy_PerHost{ + def: defaultDialer, + bypass: bypass, + } +} + +// Dial connects to the address addr on the given network through either +// defaultDialer or bypass. +func (p *proxy_PerHost) Dial(network, addr string) (c net.Conn, err error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + return p.dialerForRequest(host).Dial(network, addr) +} + +func (p *proxy_PerHost) dialerForRequest(host string) proxy_Dialer { + if ip := net.ParseIP(host); ip != nil { + for _, net := range p.bypassNetworks { + if net.Contains(ip) { + return p.bypass + } + } + for _, bypassIP := range p.bypassIPs { + if bypassIP.Equal(ip) { + return p.bypass + } + } + return p.def + } + + for _, zone := range p.bypassZones { + if strings.HasSuffix(host, zone) { + return p.bypass + } + if host == zone[1:] { + // For a zone ".example.com", we match "example.com" + // too. + return p.bypass + } + } + for _, bypassHost := range p.bypassHosts { + if bypassHost == host { + return p.bypass + } + } + return p.def +} + +// AddFromString parses a string that contains comma-separated values +// specifying hosts that should use the bypass proxy. Each value is either an +// IP address, a CIDR range, a zone (*.example.com) or a host name +// (localhost). A best effort is made to parse the string and errors are +// ignored. +func (p *proxy_PerHost) AddFromString(s string) { + hosts := strings.Split(s, ",") + for _, host := range hosts { + host = strings.TrimSpace(host) + if len(host) == 0 { + continue + } + if strings.Contains(host, "/") { + // We assume that it's a CIDR address like 127.0.0.0/8 + if _, net, err := net.ParseCIDR(host); err == nil { + p.AddNetwork(net) + } + continue + } + if ip := net.ParseIP(host); ip != nil { + p.AddIP(ip) + continue + } + if strings.HasPrefix(host, "*.") { + p.AddZone(host[1:]) + continue + } + p.AddHost(host) + } +} + +// AddIP specifies an IP address that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match an IP. +func (p *proxy_PerHost) AddIP(ip net.IP) { + p.bypassIPs = append(p.bypassIPs, ip) +} + +// AddNetwork specifies an IP range that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match. +func (p *proxy_PerHost) AddNetwork(net *net.IPNet) { + p.bypassNetworks = append(p.bypassNetworks, net) +} + +// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of +// "example.com" matches "example.com" and all of its subdomains. +func (p *proxy_PerHost) AddZone(zone string) { + if strings.HasSuffix(zone, ".") { + zone = zone[:len(zone)-1] + } + if !strings.HasPrefix(zone, ".") { + zone = "." + zone + } + p.bypassZones = append(p.bypassZones, zone) +} + +// AddHost specifies a host name that will use the bypass proxy. +func (p *proxy_PerHost) AddHost(host string) { + if strings.HasSuffix(host, ".") { + host = host[:len(host)-1] + } + p.bypassHosts = append(p.bypassHosts, host) +} + +// A Dialer is a means to establish a connection. +type proxy_Dialer interface { + // Dial connects to the given address via the proxy. + Dial(network, addr string) (c net.Conn, err error) +} + +// Auth contains authentication parameters that specific Dialers may require. +type proxy_Auth struct { + User, Password string +} + +// FromEnvironment returns the dialer specified by the proxy related variables in +// the environment. +func proxy_FromEnvironment() proxy_Dialer { + allProxy := proxy_allProxyEnv.Get() + if len(allProxy) == 0 { + return proxy_Direct + } + + proxyURL, err := url.Parse(allProxy) + if err != nil { + return proxy_Direct + } + proxy, err := proxy_FromURL(proxyURL, proxy_Direct) + if err != nil { + return proxy_Direct + } + + noProxy := proxy_noProxyEnv.Get() + if len(noProxy) == 0 { + return proxy + } + + perHost := proxy_NewPerHost(proxy, proxy_Direct) + perHost.AddFromString(noProxy) + return perHost +} + +// proxySchemes is a map from URL schemes to a function that creates a Dialer +// from a URL with such a scheme. +var proxy_proxySchemes map[string]func(*url.URL, proxy_Dialer) (proxy_Dialer, error) + +// RegisterDialerType takes a URL scheme and a function to generate Dialers from +// a URL with that scheme and a forwarding Dialer. Registered schemes are used +// by FromURL. +func proxy_RegisterDialerType(scheme string, f func(*url.URL, proxy_Dialer) (proxy_Dialer, error)) { + if proxy_proxySchemes == nil { + proxy_proxySchemes = make(map[string]func(*url.URL, proxy_Dialer) (proxy_Dialer, error)) + } + proxy_proxySchemes[scheme] = f +} + +// FromURL returns a Dialer given a URL specification and an underlying +// Dialer for it to make network requests. +func proxy_FromURL(u *url.URL, forward proxy_Dialer) (proxy_Dialer, error) { + var auth *proxy_Auth + if u.User != nil { + auth = new(proxy_Auth) + auth.User = u.User.Username() + if p, ok := u.User.Password(); ok { + auth.Password = p + } + } + + switch u.Scheme { + case "socks5": + return proxy_SOCKS5("tcp", u.Host, auth, forward) + } + + // If the scheme doesn't match any of the built-in schemes, see if it + // was registered by another package. + if proxy_proxySchemes != nil { + if f, ok := proxy_proxySchemes[u.Scheme]; ok { + return f(u, forward) + } + } + + return nil, errors.New("proxy: unknown scheme: " + u.Scheme) +} + +var ( + proxy_allProxyEnv = &proxy_envOnce{ + names: []string{"ALL_PROXY", "all_proxy"}, + } + proxy_noProxyEnv = &proxy_envOnce{ + names: []string{"NO_PROXY", "no_proxy"}, + } +) + +// envOnce looks up an environment variable (optionally by multiple +// names) once. It mitigates expensive lookups on some platforms +// (e.g. Windows). +// (Borrowed from net/http/transport.go) +type proxy_envOnce struct { + names []string + once sync.Once + val string +} + +func (e *proxy_envOnce) Get() string { + e.once.Do(e.init) + return e.val +} + +func (e *proxy_envOnce) init() { + for _, n := range e.names { + e.val = os.Getenv(n) + if e.val != "" { + return + } + } +} + +// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given address +// with an optional username and password. See RFC 1928 and RFC 1929. +func proxy_SOCKS5(network, addr string, auth *proxy_Auth, forward proxy_Dialer) (proxy_Dialer, error) { + s := &proxy_socks5{ + network: network, + addr: addr, + forward: forward, + } + if auth != nil { + s.user = auth.User + s.password = auth.Password + } + + return s, nil +} + +type proxy_socks5 struct { + user, password string + network, addr string + forward proxy_Dialer +} + +const proxy_socks5Version = 5 + +const ( + proxy_socks5AuthNone = 0 + proxy_socks5AuthPassword = 2 +) + +const proxy_socks5Connect = 1 + +const ( + proxy_socks5IP4 = 1 + proxy_socks5Domain = 3 + proxy_socks5IP6 = 4 +) + +var proxy_socks5Errors = []string{ + "", + "general failure", + "connection forbidden", + "network unreachable", + "host unreachable", + "connection refused", + "TTL expired", + "command not supported", + "address type not supported", +} + +// Dial connects to the address addr on the given network via the SOCKS5 proxy. +func (s *proxy_socks5) Dial(network, addr string) (net.Conn, error) { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return nil, errors.New("proxy: no support for SOCKS5 proxy connections of type " + network) + } + + conn, err := s.forward.Dial(s.network, s.addr) + if err != nil { + return nil, err + } + if err := s.connect(conn, addr); err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + +// connect takes an existing connection to a socks5 proxy server, +// and commands the server to extend that connection to target, +// which must be a canonical address with a host and port. +func (s *proxy_socks5) connect(conn net.Conn, target string) error { + host, portStr, err := net.SplitHostPort(target) + if err != nil { + return err + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return errors.New("proxy: failed to parse port number: " + portStr) + } + if port < 1 || port > 0xffff { + return errors.New("proxy: port number out of range: " + portStr) + } + + // the size here is just an estimate + buf := make([]byte, 0, 6+len(host)) + + buf = append(buf, proxy_socks5Version) + if len(s.user) > 0 && len(s.user) < 256 && len(s.password) < 256 { + buf = append(buf, 2 /* num auth methods */, proxy_socks5AuthNone, proxy_socks5AuthPassword) + } else { + buf = append(buf, 1 /* num auth methods */, proxy_socks5AuthNone) + } + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write greeting to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read greeting from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + if buf[0] != 5 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " has unexpected version " + strconv.Itoa(int(buf[0]))) + } + if buf[1] == 0xff { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " requires authentication") + } + + // See RFC 1929 + if buf[1] == proxy_socks5AuthPassword { + buf = buf[:0] + buf = append(buf, 1 /* password protocol version */) + buf = append(buf, uint8(len(s.user))) + buf = append(buf, s.user...) + buf = append(buf, uint8(len(s.password))) + buf = append(buf, s.password...) + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write authentication request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read authentication reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if buf[1] != 0 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " rejected username/password") + } + } + + buf = buf[:0] + buf = append(buf, proxy_socks5Version, proxy_socks5Connect, 0 /* reserved */) + + if ip := net.ParseIP(host); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + buf = append(buf, proxy_socks5IP4) + ip = ip4 + } else { + buf = append(buf, proxy_socks5IP6) + } + buf = append(buf, ip...) + } else { + if len(host) > 255 { + return errors.New("proxy: destination host name too long: " + host) + } + buf = append(buf, proxy_socks5Domain) + buf = append(buf, byte(len(host))) + buf = append(buf, host...) + } + buf = append(buf, byte(port>>8), byte(port)) + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write connect request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:4]); err != nil { + return errors.New("proxy: failed to read connect reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + failure := "unknown error" + if int(buf[1]) < len(proxy_socks5Errors) { + failure = proxy_socks5Errors[buf[1]] + } + + if len(failure) > 0 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " failed to connect: " + failure) + } + + bytesToDiscard := 0 + switch buf[3] { + case proxy_socks5IP4: + bytesToDiscard = net.IPv4len + case proxy_socks5IP6: + bytesToDiscard = net.IPv6len + case proxy_socks5Domain: + _, err := io.ReadFull(conn, buf[:1]) + if err != nil { + return errors.New("proxy: failed to read domain length from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + bytesToDiscard = int(buf[0]) + default: + return errors.New("proxy: got unknown address type " + strconv.Itoa(int(buf[3])) + " from SOCKS5 proxy at " + s.addr) + } + + if cap(buf) < bytesToDiscard { + buf = make([]byte, bytesToDiscard) + } else { + buf = buf[:bytesToDiscard] + } + if _, err := io.ReadFull(conn, buf); err != nil { + return errors.New("proxy: failed to read address from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + // Also need to discard the port number + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read port from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + return nil +} diff --git a/vendor/github.com/hashicorp/hcl/appveyor.yml b/vendor/github.com/hashicorp/hcl/appveyor.yml index 3c8cdf8e..4db0b711 100644 --- a/vendor/github.com/hashicorp/hcl/appveyor.yml +++ b/vendor/github.com/hashicorp/hcl/appveyor.yml @@ -4,7 +4,7 @@ clone_folder: c:\gopath\src\github.com\hashicorp\hcl environment: GOPATH: c:\gopath init: - - git config --global core.autocrlf true + - git config --global core.autocrlf false install: - cmd: >- echo %Path% diff --git a/vendor/github.com/hashicorp/hcl/decoder.go b/vendor/github.com/hashicorp/hcl/decoder.go index 0b39c1b9..bed9ebbe 100644 --- a/vendor/github.com/hashicorp/hcl/decoder.go +++ b/vendor/github.com/hashicorp/hcl/decoder.go @@ -89,7 +89,7 @@ func (d *decoder) decode(name string, node ast.Node, result reflect.Value) error switch k.Kind() { case reflect.Bool: return d.decodeBool(name, node, result) - case reflect.Float64: + case reflect.Float32, reflect.Float64: return d.decodeFloat(name, node, result) case reflect.Int, reflect.Int32, reflect.Int64: return d.decodeInt(name, node, result) @@ -137,13 +137,13 @@ func (d *decoder) decodeBool(name string, node ast.Node, result reflect.Value) e func (d *decoder) decodeFloat(name string, node ast.Node, result reflect.Value) error { switch n := node.(type) { case *ast.LiteralType: - if n.Token.Type == token.FLOAT { + if n.Token.Type == token.FLOAT || n.Token.Type == token.NUMBER { v, err := strconv.ParseFloat(n.Token.Text, 64) if err != nil { return err } - result.Set(reflect.ValueOf(v)) + result.Set(reflect.ValueOf(v).Convert(result.Type())) return nil } } @@ -573,7 +573,11 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) // Compile the list of all the fields that we're going to be decoding // from all the structs. - fields := make(map[*reflect.StructField]reflect.Value) + type field struct { + field reflect.StructField + val reflect.Value + } + fields := []field{} for len(structs) > 0 { structVal := structs[0] structs = structs[1:] @@ -616,7 +620,7 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) } // Normal struct field, store it away - fields[&fieldType] = structVal.Field(i) + fields = append(fields, field{fieldType, structVal.Field(i)}) } } @@ -624,26 +628,27 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) decodedFields := make([]string, 0, len(fields)) decodedFieldsVal := make([]reflect.Value, 0) unusedKeysVal := make([]reflect.Value, 0) - for fieldType, field := range fields { - if !field.IsValid() { + for _, f := range fields { + field, fieldValue := f.field, f.val + if !fieldValue.IsValid() { // This should never happen panic("field is not valid") } // If we can't set the field, then it is unexported or something, // and we just continue onwards. - if !field.CanSet() { + if !fieldValue.CanSet() { continue } - fieldName := fieldType.Name + fieldName := field.Name - tagValue := fieldType.Tag.Get(tagName) + tagValue := field.Tag.Get(tagName) tagParts := strings.SplitN(tagValue, ",", 2) if len(tagParts) >= 2 { switch tagParts[1] { case "decodedFields": - decodedFieldsVal = append(decodedFieldsVal, field) + decodedFieldsVal = append(decodedFieldsVal, fieldValue) continue case "key": if item == nil { @@ -654,10 +659,10 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) } } - field.SetString(item.Keys[0].Token.Value().(string)) + fieldValue.SetString(item.Keys[0].Token.Value().(string)) continue case "unusedKeys": - unusedKeysVal = append(unusedKeysVal, field) + unusedKeysVal = append(unusedKeysVal, fieldValue) continue } } @@ -684,7 +689,7 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) // because we actually want the value. fieldName = fmt.Sprintf("%s.%s", name, fieldName) if len(prefixMatches.Items) > 0 { - if err := d.decode(fieldName, prefixMatches, field); err != nil { + if err := d.decode(fieldName, prefixMatches, fieldValue); err != nil { return err } } @@ -694,12 +699,12 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) decodeNode = &ast.ObjectList{Items: ot.List.Items} } - if err := d.decode(fieldName, decodeNode, field); err != nil { + if err := d.decode(fieldName, decodeNode, fieldValue); err != nil { return err } } - decodedFields = append(decodedFields, fieldType.Name) + decodedFields = append(decodedFields, field.Name) } if len(decodedFieldsVal) > 0 { diff --git a/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go b/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go index 6e54bed9..64c83bcf 100644 --- a/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go +++ b/vendor/github.com/hashicorp/hcl/hcl/parser/parser.go @@ -3,6 +3,7 @@ package parser import ( + "bytes" "errors" "fmt" "strings" @@ -36,6 +37,11 @@ func newParser(src []byte) *Parser { // Parse returns the fully parsed source and returns the abstract syntax tree. func Parse(src []byte) (*ast.File, error) { + // normalize all line endings + // since the scanner and output only work with "\n" line endings, we may + // end up with dangling "\r" characters in the parsed data. + src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1) + p := newParser(src) return p.Parse() } @@ -191,9 +197,18 @@ func (p *Parser) objectItem() (*ast.ObjectItem, error) { keyStr = append(keyStr, k.Token.Text) } - return nil, fmt.Errorf( - "key '%s' expected start of object ('{') or assignment ('=')", - strings.Join(keyStr, " ")) + return nil, &PosError{ + Pos: p.tok.Pos, + Err: fmt.Errorf( + "key '%s' expected start of object ('{') or assignment ('=')", + strings.Join(keyStr, " ")), + } + } + + // key=#comment + // val + if p.lineComment != nil { + o.LineComment, p.lineComment = p.lineComment, nil } // do a look-ahead for line comment @@ -313,7 +328,10 @@ func (p *Parser) objectType() (*ast.ObjectType, error) { // No error, scan and expect the ending to be a brace if tok := p.scan(); tok.Type != token.RBRACE { - return nil, fmt.Errorf("object expected closing RBRACE got: %s", tok.Type) + return nil, &PosError{ + Pos: tok.Pos, + Err: fmt.Errorf("object expected closing RBRACE got: %s", tok.Type), + } } o.List = l diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/nodes.go b/vendor/github.com/hashicorp/hcl/hcl/printer/nodes.go new file mode 100644 index 00000000..7c038d12 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/printer/nodes.go @@ -0,0 +1,789 @@ +package printer + +import ( + "bytes" + "fmt" + "sort" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/token" +) + +const ( + blank = byte(' ') + newline = byte('\n') + tab = byte('\t') + infinity = 1 << 30 // offset or line +) + +var ( + unindent = []byte("\uE123") // in the private use space +) + +type printer struct { + cfg Config + prev token.Pos + + comments []*ast.CommentGroup // may be nil, contains all comments + standaloneComments []*ast.CommentGroup // contains all standalone comments (not assigned to any node) + + enableTrace bool + indentTrace int +} + +type ByPosition []*ast.CommentGroup + +func (b ByPosition) Len() int { return len(b) } +func (b ByPosition) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b ByPosition) Less(i, j int) bool { return b[i].Pos().Before(b[j].Pos()) } + +// collectComments comments all standalone comments which are not lead or line +// comment +func (p *printer) collectComments(node ast.Node) { + // first collect all comments. This is already stored in + // ast.File.(comments) + ast.Walk(node, func(nn ast.Node) (ast.Node, bool) { + switch t := nn.(type) { + case *ast.File: + p.comments = t.Comments + return nn, false + } + return nn, true + }) + + standaloneComments := make(map[token.Pos]*ast.CommentGroup, 0) + for _, c := range p.comments { + standaloneComments[c.Pos()] = c + } + + // next remove all lead and line comments from the overall comment map. + // This will give us comments which are standalone, comments which are not + // assigned to any kind of node. + ast.Walk(node, func(nn ast.Node) (ast.Node, bool) { + switch t := nn.(type) { + case *ast.LiteralType: + if t.LeadComment != nil { + for _, comment := range t.LeadComment.List { + if _, ok := standaloneComments[comment.Pos()]; ok { + delete(standaloneComments, comment.Pos()) + } + } + } + + if t.LineComment != nil { + for _, comment := range t.LineComment.List { + if _, ok := standaloneComments[comment.Pos()]; ok { + delete(standaloneComments, comment.Pos()) + } + } + } + case *ast.ObjectItem: + if t.LeadComment != nil { + for _, comment := range t.LeadComment.List { + if _, ok := standaloneComments[comment.Pos()]; ok { + delete(standaloneComments, comment.Pos()) + } + } + } + + if t.LineComment != nil { + for _, comment := range t.LineComment.List { + if _, ok := standaloneComments[comment.Pos()]; ok { + delete(standaloneComments, comment.Pos()) + } + } + } + } + + return nn, true + }) + + for _, c := range standaloneComments { + p.standaloneComments = append(p.standaloneComments, c) + } + + sort.Sort(ByPosition(p.standaloneComments)) +} + +// output prints creates b printable HCL output and returns it. +func (p *printer) output(n interface{}) []byte { + var buf bytes.Buffer + + switch t := n.(type) { + case *ast.File: + // File doesn't trace so we add the tracing here + defer un(trace(p, "File")) + return p.output(t.Node) + case *ast.ObjectList: + defer un(trace(p, "ObjectList")) + + var index int + for { + // Determine the location of the next actual non-comment + // item. If we're at the end, the next item is at "infinity" + var nextItem token.Pos + if index != len(t.Items) { + nextItem = t.Items[index].Pos() + } else { + nextItem = token.Pos{Offset: infinity, Line: infinity} + } + + // Go through the standalone comments in the file and print out + // the comments that we should be for this object item. + for _, c := range p.standaloneComments { + // Go through all the comments in the group. The group + // should be printed together, not separated by double newlines. + printed := false + newlinePrinted := false + for _, comment := range c.List { + // We only care about comments after the previous item + // we've printed so that comments are printed in the + // correct locations (between two objects for example). + // And before the next item. + if comment.Pos().After(p.prev) && comment.Pos().Before(nextItem) { + // if we hit the end add newlines so we can print the comment + // we don't do this if prev is invalid which means the + // beginning of the file since the first comment should + // be at the first line. + if !newlinePrinted && p.prev.IsValid() && index == len(t.Items) { + buf.Write([]byte{newline, newline}) + newlinePrinted = true + } + + // Write the actual comment. + buf.WriteString(comment.Text) + buf.WriteByte(newline) + + // Set printed to true to note that we printed something + printed = true + } + } + + // If we're not at the last item, write a new line so + // that there is a newline separating this comment from + // the next object. + if printed && index != len(t.Items) { + buf.WriteByte(newline) + } + } + + if index == len(t.Items) { + break + } + + buf.Write(p.output(t.Items[index])) + if index != len(t.Items)-1 { + // Always write a newline to separate us from the next item + buf.WriteByte(newline) + + // Need to determine if we're going to separate the next item + // with a blank line. The logic here is simple, though there + // are a few conditions: + // + // 1. The next object is more than one line away anyways, + // so we need an empty line. + // + // 2. The next object is not a "single line" object, so + // we need an empty line. + // + // 3. This current object is not a single line object, + // so we need an empty line. + current := t.Items[index] + next := t.Items[index+1] + if next.Pos().Line != t.Items[index].Pos().Line+1 || + !p.isSingleLineObject(next) || + !p.isSingleLineObject(current) { + buf.WriteByte(newline) + } + } + index++ + } + case *ast.ObjectKey: + buf.WriteString(t.Token.Text) + case *ast.ObjectItem: + p.prev = t.Pos() + buf.Write(p.objectItem(t)) + case *ast.LiteralType: + buf.Write(p.literalType(t)) + case *ast.ListType: + buf.Write(p.list(t)) + case *ast.ObjectType: + buf.Write(p.objectType(t)) + default: + fmt.Printf(" unknown type: %T\n", n) + } + + return buf.Bytes() +} + +func (p *printer) literalType(lit *ast.LiteralType) []byte { + result := []byte(lit.Token.Text) + switch lit.Token.Type { + case token.HEREDOC: + // Clear the trailing newline from heredocs + if result[len(result)-1] == '\n' { + result = result[:len(result)-1] + } + + // Poison lines 2+ so that we don't indent them + result = p.heredocIndent(result) + case token.STRING: + // If this is a multiline string, poison lines 2+ so we don't + // indent them. + if bytes.IndexRune(result, '\n') >= 0 { + result = p.heredocIndent(result) + } + } + + return result +} + +// objectItem returns the printable HCL form of an object item. An object type +// starts with one/multiple keys and has a value. The value might be of any +// type. +func (p *printer) objectItem(o *ast.ObjectItem) []byte { + defer un(trace(p, fmt.Sprintf("ObjectItem: %s", o.Keys[0].Token.Text))) + var buf bytes.Buffer + + if o.LeadComment != nil { + for _, comment := range o.LeadComment.List { + buf.WriteString(comment.Text) + buf.WriteByte(newline) + } + } + + // If key and val are on different lines, treat line comments like lead comments. + if o.LineComment != nil && o.Val.Pos().Line != o.Keys[0].Pos().Line { + for _, comment := range o.LineComment.List { + buf.WriteString(comment.Text) + buf.WriteByte(newline) + } + } + + for i, k := range o.Keys { + buf.WriteString(k.Token.Text) + buf.WriteByte(blank) + + // reach end of key + if o.Assign.IsValid() && i == len(o.Keys)-1 && len(o.Keys) == 1 { + buf.WriteString("=") + buf.WriteByte(blank) + } + } + + buf.Write(p.output(o.Val)) + + if o.LineComment != nil && o.Val.Pos().Line == o.Keys[0].Pos().Line { + buf.WriteByte(blank) + for _, comment := range o.LineComment.List { + buf.WriteString(comment.Text) + } + } + + return buf.Bytes() +} + +// objectType returns the printable HCL form of an object type. An object type +// begins with a brace and ends with a brace. +func (p *printer) objectType(o *ast.ObjectType) []byte { + defer un(trace(p, "ObjectType")) + var buf bytes.Buffer + buf.WriteString("{") + + var index int + var nextItem token.Pos + var commented, newlinePrinted bool + for { + // Determine the location of the next actual non-comment + // item. If we're at the end, the next item is the closing brace + if index != len(o.List.Items) { + nextItem = o.List.Items[index].Pos() + } else { + nextItem = o.Rbrace + } + + // Go through the standalone comments in the file and print out + // the comments that we should be for this object item. + for _, c := range p.standaloneComments { + printed := false + var lastCommentPos token.Pos + for _, comment := range c.List { + // We only care about comments after the previous item + // we've printed so that comments are printed in the + // correct locations (between two objects for example). + // And before the next item. + if comment.Pos().After(p.prev) && comment.Pos().Before(nextItem) { + // If there are standalone comments and the initial newline has not + // been printed yet, do it now. + if !newlinePrinted { + newlinePrinted = true + buf.WriteByte(newline) + } + + // add newline if it's between other printed nodes + if index > 0 { + commented = true + buf.WriteByte(newline) + } + + // Store this position + lastCommentPos = comment.Pos() + + // output the comment itself + buf.Write(p.indent(p.heredocIndent([]byte(comment.Text)))) + + // Set printed to true to note that we printed something + printed = true + + /* + if index != len(o.List.Items) { + buf.WriteByte(newline) // do not print on the end + } + */ + } + } + + // Stuff to do if we had comments + if printed { + // Always write a newline + buf.WriteByte(newline) + + // If there is another item in the object and our comment + // didn't hug it directly, then make sure there is a blank + // line separating them. + if nextItem != o.Rbrace && nextItem.Line != lastCommentPos.Line+1 { + buf.WriteByte(newline) + } + } + } + + if index == len(o.List.Items) { + p.prev = o.Rbrace + break + } + + // At this point we are sure that it's not a totally empty block: print + // the initial newline if it hasn't been printed yet by the previous + // block about standalone comments. + if !newlinePrinted { + buf.WriteByte(newline) + newlinePrinted = true + } + + // check if we have adjacent one liner items. If yes we'll going to align + // the comments. + var aligned []*ast.ObjectItem + for _, item := range o.List.Items[index:] { + // we don't group one line lists + if len(o.List.Items) == 1 { + break + } + + // one means a oneliner with out any lead comment + // two means a oneliner with lead comment + // anything else might be something else + cur := lines(string(p.objectItem(item))) + if cur > 2 { + break + } + + curPos := item.Pos() + + nextPos := token.Pos{} + if index != len(o.List.Items)-1 { + nextPos = o.List.Items[index+1].Pos() + } + + prevPos := token.Pos{} + if index != 0 { + prevPos = o.List.Items[index-1].Pos() + } + + // fmt.Println("DEBUG ----------------") + // fmt.Printf("prev = %+v prevPos: %s\n", prev, prevPos) + // fmt.Printf("cur = %+v curPos: %s\n", cur, curPos) + // fmt.Printf("next = %+v nextPos: %s\n", next, nextPos) + + if curPos.Line+1 == nextPos.Line { + aligned = append(aligned, item) + index++ + continue + } + + if curPos.Line-1 == prevPos.Line { + aligned = append(aligned, item) + index++ + + // finish if we have a new line or comment next. This happens + // if the next item is not adjacent + if curPos.Line+1 != nextPos.Line { + break + } + continue + } + + break + } + + // put newlines if the items are between other non aligned items. + // newlines are also added if there is a standalone comment already, so + // check it too + if !commented && index != len(aligned) { + buf.WriteByte(newline) + } + + if len(aligned) >= 1 { + p.prev = aligned[len(aligned)-1].Pos() + + items := p.alignedItems(aligned) + buf.Write(p.indent(items)) + } else { + p.prev = o.List.Items[index].Pos() + + buf.Write(p.indent(p.objectItem(o.List.Items[index]))) + index++ + } + + buf.WriteByte(newline) + } + + buf.WriteString("}") + return buf.Bytes() +} + +func (p *printer) alignedItems(items []*ast.ObjectItem) []byte { + var buf bytes.Buffer + + // find the longest key and value length, needed for alignment + var longestKeyLen int // longest key length + var longestValLen int // longest value length + for _, item := range items { + key := len(item.Keys[0].Token.Text) + val := len(p.output(item.Val)) + + if key > longestKeyLen { + longestKeyLen = key + } + + if val > longestValLen { + longestValLen = val + } + } + + for i, item := range items { + if item.LeadComment != nil { + for _, comment := range item.LeadComment.List { + buf.WriteString(comment.Text) + buf.WriteByte(newline) + } + } + + for i, k := range item.Keys { + keyLen := len(k.Token.Text) + buf.WriteString(k.Token.Text) + for i := 0; i < longestKeyLen-keyLen+1; i++ { + buf.WriteByte(blank) + } + + // reach end of key + if i == len(item.Keys)-1 && len(item.Keys) == 1 { + buf.WriteString("=") + buf.WriteByte(blank) + } + } + + val := p.output(item.Val) + valLen := len(val) + buf.Write(val) + + if item.Val.Pos().Line == item.Keys[0].Pos().Line && item.LineComment != nil { + for i := 0; i < longestValLen-valLen+1; i++ { + buf.WriteByte(blank) + } + + for _, comment := range item.LineComment.List { + buf.WriteString(comment.Text) + } + } + + // do not print for the last item + if i != len(items)-1 { + buf.WriteByte(newline) + } + } + + return buf.Bytes() +} + +// list returns the printable HCL form of an list type. +func (p *printer) list(l *ast.ListType) []byte { + if p.isSingleLineList(l) { + return p.singleLineList(l) + } + + var buf bytes.Buffer + buf.WriteString("[") + buf.WriteByte(newline) + + var longestLine int + for _, item := range l.List { + // for now we assume that the list only contains literal types + if lit, ok := item.(*ast.LiteralType); ok { + lineLen := len(lit.Token.Text) + if lineLen > longestLine { + longestLine = lineLen + } + } + } + + haveEmptyLine := false + for i, item := range l.List { + // If we have a lead comment, then we want to write that first + leadComment := false + if lit, ok := item.(*ast.LiteralType); ok && lit.LeadComment != nil { + leadComment = true + + // Ensure an empty line before every element with a + // lead comment (except the first item in a list). + if !haveEmptyLine && i != 0 { + buf.WriteByte(newline) + } + + for _, comment := range lit.LeadComment.List { + buf.Write(p.indent([]byte(comment.Text))) + buf.WriteByte(newline) + } + } + + // also indent each line + val := p.output(item) + curLen := len(val) + buf.Write(p.indent(val)) + + // if this item is a heredoc, then we output the comma on + // the next line. This is the only case this happens. + comma := []byte{','} + if lit, ok := item.(*ast.LiteralType); ok && lit.Token.Type == token.HEREDOC { + buf.WriteByte(newline) + comma = p.indent(comma) + } + + buf.Write(comma) + + if lit, ok := item.(*ast.LiteralType); ok && lit.LineComment != nil { + // if the next item doesn't have any comments, do not align + buf.WriteByte(blank) // align one space + for i := 0; i < longestLine-curLen; i++ { + buf.WriteByte(blank) + } + + for _, comment := range lit.LineComment.List { + buf.WriteString(comment.Text) + } + } + + buf.WriteByte(newline) + + // Ensure an empty line after every element with a + // lead comment (except the first item in a list). + haveEmptyLine = leadComment && i != len(l.List)-1 + if haveEmptyLine { + buf.WriteByte(newline) + } + } + + buf.WriteString("]") + return buf.Bytes() +} + +// isSingleLineList returns true if: +// * they were previously formatted entirely on one line +// * they consist entirely of literals +// * there are either no heredoc strings or the list has exactly one element +// * there are no line comments +func (printer) isSingleLineList(l *ast.ListType) bool { + for _, item := range l.List { + if item.Pos().Line != l.Lbrack.Line { + return false + } + + lit, ok := item.(*ast.LiteralType) + if !ok { + return false + } + + if lit.Token.Type == token.HEREDOC && len(l.List) != 1 { + return false + } + + if lit.LineComment != nil { + return false + } + } + + return true +} + +// singleLineList prints a simple single line list. +// For a definition of "simple", see isSingleLineList above. +func (p *printer) singleLineList(l *ast.ListType) []byte { + buf := &bytes.Buffer{} + + buf.WriteString("[") + for i, item := range l.List { + if i != 0 { + buf.WriteString(", ") + } + + // Output the item itself + buf.Write(p.output(item)) + + // The heredoc marker needs to be at the end of line. + if lit, ok := item.(*ast.LiteralType); ok && lit.Token.Type == token.HEREDOC { + buf.WriteByte(newline) + } + } + + buf.WriteString("]") + return buf.Bytes() +} + +// indent indents the lines of the given buffer for each non-empty line +func (p *printer) indent(buf []byte) []byte { + var prefix []byte + if p.cfg.SpacesWidth != 0 { + for i := 0; i < p.cfg.SpacesWidth; i++ { + prefix = append(prefix, blank) + } + } else { + prefix = []byte{tab} + } + + var res []byte + bol := true + for _, c := range buf { + if bol && c != '\n' { + res = append(res, prefix...) + } + + res = append(res, c) + bol = c == '\n' + } + return res +} + +// unindent removes all the indentation from the tombstoned lines +func (p *printer) unindent(buf []byte) []byte { + var res []byte + for i := 0; i < len(buf); i++ { + skip := len(buf)-i <= len(unindent) + if !skip { + skip = !bytes.Equal(unindent, buf[i:i+len(unindent)]) + } + if skip { + res = append(res, buf[i]) + continue + } + + // We have a marker. we have to backtrace here and clean out + // any whitespace ahead of our tombstone up to a \n + for j := len(res) - 1; j >= 0; j-- { + if res[j] == '\n' { + break + } + + res = res[:j] + } + + // Skip the entire unindent marker + i += len(unindent) - 1 + } + + return res +} + +// heredocIndent marks all the 2nd and further lines as unindentable +func (p *printer) heredocIndent(buf []byte) []byte { + var res []byte + bol := false + for _, c := range buf { + if bol && c != '\n' { + res = append(res, unindent...) + } + res = append(res, c) + bol = c == '\n' + } + return res +} + +// isSingleLineObject tells whether the given object item is a single +// line object such as "obj {}". +// +// A single line object: +// +// * has no lead comments (hence multi-line) +// * has no assignment +// * has no values in the stanza (within {}) +// +func (p *printer) isSingleLineObject(val *ast.ObjectItem) bool { + // If there is a lead comment, can't be one line + if val.LeadComment != nil { + return false + } + + // If there is assignment, we always break by line + if val.Assign.IsValid() { + return false + } + + // If it isn't an object type, then its not a single line object + ot, ok := val.Val.(*ast.ObjectType) + if !ok { + return false + } + + // If the object has no items, it is single line! + return len(ot.List.Items) == 0 +} + +func lines(txt string) int { + endline := 1 + for i := 0; i < len(txt); i++ { + if txt[i] == '\n' { + endline++ + } + } + return endline +} + +// ---------------------------------------------------------------------------- +// Tracing support + +func (p *printer) printTrace(a ...interface{}) { + if !p.enableTrace { + return + } + + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = len(dots) + i := 2 * p.indentTrace + for i > n { + fmt.Print(dots) + i -= n + } + // i <= n + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *printer, msg string) *printer { + p.printTrace(msg, "(") + p.indentTrace++ + return p +} + +// Usage pattern: defer un(trace(p, "...")) +func un(p *printer) { + p.indentTrace-- + p.printTrace(")") +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go b/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go new file mode 100644 index 00000000..6617ab8e --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/hcl/printer/printer.go @@ -0,0 +1,66 @@ +// Package printer implements printing of AST nodes to HCL format. +package printer + +import ( + "bytes" + "io" + "text/tabwriter" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/parser" +) + +var DefaultConfig = Config{ + SpacesWidth: 2, +} + +// A Config node controls the output of Fprint. +type Config struct { + SpacesWidth int // if set, it will use spaces instead of tabs for alignment +} + +func (c *Config) Fprint(output io.Writer, node ast.Node) error { + p := &printer{ + cfg: *c, + comments: make([]*ast.CommentGroup, 0), + standaloneComments: make([]*ast.CommentGroup, 0), + // enableTrace: true, + } + + p.collectComments(node) + + if _, err := output.Write(p.unindent(p.output(node))); err != nil { + return err + } + + // flush tabwriter, if any + var err error + if tw, _ := output.(*tabwriter.Writer); tw != nil { + err = tw.Flush() + } + + return err +} + +// Fprint "pretty-prints" an HCL node to output +// It calls Config.Fprint with default settings. +func Fprint(output io.Writer, node ast.Node) error { + return DefaultConfig.Fprint(output, node) +} + +// Format formats src HCL and returns the result. +func Format(src []byte) ([]byte, error) { + node, err := parser.Parse(src) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if err := DefaultConfig.Fprint(&buf, node); err != nil { + return nil, err + } + + // Add trailing newline to result + buf.WriteString("\n") + return buf.Bytes(), nil +} diff --git a/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner.go b/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner.go index 69662367..624a18fe 100644 --- a/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner.go +++ b/vendor/github.com/hashicorp/hcl/hcl/scanner/scanner.go @@ -74,14 +74,6 @@ func (s *Scanner) next() rune { return eof } - if ch == utf8.RuneError && size == 1 { - s.srcPos.Column++ - s.srcPos.Offset += size - s.lastCharLen = size - s.err("illegal UTF-8 encoding") - return ch - } - // remember last position s.prevPos = s.srcPos @@ -89,18 +81,27 @@ func (s *Scanner) next() rune { s.lastCharLen = size s.srcPos.Offset += size + if ch == utf8.RuneError && size == 1 { + s.err("illegal UTF-8 encoding") + return ch + } + if ch == '\n' { s.srcPos.Line++ s.lastLineLen = s.srcPos.Column s.srcPos.Column = 0 } - // If we see a null character with data left, then that is an error - if ch == '\x00' && s.buf.Len() > 0 { + if ch == '\x00' { s.err("unexpected null character (0x00)") return eof } + if ch == '\uE123' { + s.err("unicode code point U+E123 reserved for internal use") + return utf8.RuneError + } + // debug // fmt.Printf("ch: %q, offset:column: %d:%d\n", ch, s.srcPos.Offset, s.srcPos.Column) return ch @@ -351,7 +352,7 @@ func (s *Scanner) scanNumber(ch rune) token.Type { return token.NUMBER } -// scanMantissa scans the mantissa begining from the rune. It returns the next +// scanMantissa scans the mantissa beginning from the rune. It returns the next // non decimal rune. It's used to determine wheter it's a fraction or exponent. func (s *Scanner) scanMantissa(ch rune) rune { scanned := false @@ -432,16 +433,16 @@ func (s *Scanner) scanHeredoc() { // Read the identifier identBytes := s.src[offs : s.srcPos.Offset-s.lastCharLen] - if len(identBytes) == 0 { + if len(identBytes) == 0 || (len(identBytes) == 1 && identBytes[0] == '-') { s.err("zero-length heredoc anchor") return } var identRegexp *regexp.Regexp if identBytes[0] == '-' { - identRegexp = regexp.MustCompile(fmt.Sprintf(`[[:space:]]*%s\z`, identBytes[1:])) + identRegexp = regexp.MustCompile(fmt.Sprintf(`^[[:space:]]*%s\r*\z`, identBytes[1:])) } else { - identRegexp = regexp.MustCompile(fmt.Sprintf(`[[:space:]]*%s\z`, identBytes)) + identRegexp = regexp.MustCompile(fmt.Sprintf(`^[[:space:]]*%s\r*\z`, identBytes)) } // Read the actual string value @@ -551,7 +552,7 @@ func (s *Scanner) scanDigits(ch rune, base, n int) rune { s.err("illegal char escape") } - if n != start { + if n != start && ch != eof { // we scanned all digits, put the last non digit char back, // only if we read anything at all s.unread() diff --git a/vendor/github.com/hashicorp/hcl/json/parser/parser.go b/vendor/github.com/hashicorp/hcl/json/parser/parser.go index 6f460853..125a5f07 100644 --- a/vendor/github.com/hashicorp/hcl/json/parser/parser.go +++ b/vendor/github.com/hashicorp/hcl/json/parser/parser.go @@ -147,7 +147,7 @@ func (p *Parser) objectKey() ([]*ast.ObjectKey, error) { // Done return keys, nil case token.ILLEGAL: - fmt.Println("illegal") + return nil, errors.New("illegal") default: return nil, fmt.Errorf("expected: STRING got: %s", p.tok.Type) } diff --git a/vendor/github.com/hashicorp/hcl/json/scanner/scanner.go b/vendor/github.com/hashicorp/hcl/json/scanner/scanner.go index dd5c72bb..fe3f0f09 100644 --- a/vendor/github.com/hashicorp/hcl/json/scanner/scanner.go +++ b/vendor/github.com/hashicorp/hcl/json/scanner/scanner.go @@ -246,7 +246,7 @@ func (s *Scanner) scanNumber(ch rune) token.Type { return token.NUMBER } -// scanMantissa scans the mantissa begining from the rune. It returns the next +// scanMantissa scans the mantissa beginning from the rune. It returns the next // non decimal rune. It's used to determine wheter it's a fraction or exponent. func (s *Scanner) scanMantissa(ch rune) rune { scanned := false diff --git a/vendor/github.com/magiconair/properties/CHANGELOG.md b/vendor/github.com/magiconair/properties/CHANGELOG.md index 1d86d0c1..adefa17e 100644 --- a/vendor/github.com/magiconair/properties/CHANGELOG.md +++ b/vendor/github.com/magiconair/properties/CHANGELOG.md @@ -1,57 +1,84 @@ ## Changelog -### Unreleased +### [1.7.6](https://github.com/magiconair/properties/tree/v1.7.6) - 14 Feb 2018 + + * [PR #29](https://github.com/magiconair/properties/pull/29): Reworked expansion logic to handle more complex cases. + + See PR for an example. + + Thanks to [@yobert](https://github.com/yobert) for the fix. + +### [1.7.5](https://github.com/magiconair/properties/tree/v1.7.5) - 13 Feb 2018 + + * [PR #28](https://github.com/magiconair/properties/pull/28): Support duplicate expansions in the same value + + Values which expand the same key multiple times (e.g. `key=${a} ${a}`) will no longer fail + with a `circular reference error`. + + Thanks to [@yobert](https://github.com/yobert) for the fix. + +### [1.7.4](https://github.com/magiconair/properties/tree/v1.7.4) - 31 Oct 2017 + + * [Issue #23](https://github.com/magiconair/properties/issues/23): Ignore blank lines with whitespaces + + * [PR #24](https://github.com/magiconair/properties/pull/24): Update keys when DisableExpansion is enabled + + Thanks to [@mgurov](https://github.com/mgurov) for the fix. + +### [1.7.3](https://github.com/magiconair/properties/tree/v1.7.3) - 10 Jul 2017 * [Issue #17](https://github.com/magiconair/properties/issues/17): Add [SetValue()](http://godoc.org/github.com/magiconair/properties#Properties.SetValue) method to set values generically * [Issue #22](https://github.com/magiconair/properties/issues/22): Add [LoadMap()](http://godoc.org/github.com/magiconair/properties#LoadMap) function to load properties from a string map -### [1.7.2](https://github.com/magiconair/properties/tags/v1.7.2) - 20 Mar 2017 +### [1.7.2](https://github.com/magiconair/properties/tree/v1.7.2) - 20 Mar 2017 * [Issue #15](https://github.com/magiconair/properties/issues/15): Drop gocheck dependency * [PR #21](https://github.com/magiconair/properties/pull/21): Add [Map()](http://godoc.org/github.com/magiconair/properties#Properties.Map) and [FilterFunc()](http://godoc.org/github.com/magiconair/properties#Properties.FilterFunc) -### [1.7.1](https://github.com/magiconair/properties/tags/v1.7.1) - 13 Jan 2017 +### [1.7.1](https://github.com/magiconair/properties/tree/v1.7.1) - 13 Jan 2017 + * [Issue #14](https://github.com/magiconair/properties/issues/14): Decouple TestLoadExpandedFile from `$USER` + * [PR #12](https://github.com/magiconair/properties/pull/12): Load from files and URLs * [PR #16](https://github.com/magiconair/properties/pull/16): Keep gofmt happy * [PR #18](https://github.com/magiconair/properties/pull/18): Fix Delete() function -### [1.7.0](https://github.com/magiconair/properties/tags/v1.7.0) - 20 Mar 2016 +### [1.7.0](https://github.com/magiconair/properties/tree/v1.7.0) - 20 Mar 2016 * [Issue #10](https://github.com/magiconair/properties/issues/10): Add [LoadURL,LoadURLs,MustLoadURL,MustLoadURLs](http://godoc.org/github.com/magiconair/properties#LoadURL) method to load properties from a URL. * [Issue #11](https://github.com/magiconair/properties/issues/11): Add [LoadString,MustLoadString](http://godoc.org/github.com/magiconair/properties#LoadString) method to load properties from an UTF8 string. * [PR #8](https://github.com/magiconair/properties/pull/8): Add [MustFlag](http://godoc.org/github.com/magiconair/properties#Properties.MustFlag) method to provide overrides via command line flags. (@pascaldekloe) -### [1.6.0](https://github.com/magiconair/properties/tags/v1.6.0) - 11 Dec 2015 +### [1.6.0](https://github.com/magiconair/properties/tree/v1.6.0) - 11 Dec 2015 * Add [Decode](http://godoc.org/github.com/magiconair/properties#Properties.Decode) method to populate struct from properties via tags. -### [1.5.6](https://github.com/magiconair/properties/tags/v1.5.6) - 18 Oct 2015 +### [1.5.6](https://github.com/magiconair/properties/tree/v1.5.6) - 18 Oct 2015 * Vendored in gopkg.in/check.v1 -### [1.5.5](https://github.com/magiconair/properties/tags/v1.5.5) - 31 Jul 2015 +### [1.5.5](https://github.com/magiconair/properties/tree/v1.5.5) - 31 Jul 2015 * [PR #6](https://github.com/magiconair/properties/pull/6): Add [Delete](http://godoc.org/github.com/magiconair/properties#Properties.Delete) method to remove keys including comments. (@gerbenjacobs) -### [1.5.4](https://github.com/magiconair/properties/tags/v1.5.4) - 23 Jun 2015 +### [1.5.4](https://github.com/magiconair/properties/tree/v1.5.4) - 23 Jun 2015 * [Issue #5](https://github.com/magiconair/properties/issues/5): Allow disabling of property expansion [DisableExpansion](http://godoc.org/github.com/magiconair/properties#Properties.DisableExpansion). When property expansion is disabled Properties become a simple key/value store and don't check for circular references. -### [1.5.3](https://github.com/magiconair/properties/tags/v1.5.3) - 02 Jun 2015 +### [1.5.3](https://github.com/magiconair/properties/tree/v1.5.3) - 02 Jun 2015 * [Issue #4](https://github.com/magiconair/properties/issues/4): Maintain key order in [Filter()](http://godoc.org/github.com/magiconair/properties#Properties.Filter), [FilterPrefix()](http://godoc.org/github.com/magiconair/properties#Properties.FilterPrefix) and [FilterRegexp()](http://godoc.org/github.com/magiconair/properties#Properties.FilterRegexp) -### [1.5.2](https://github.com/magiconair/properties/tags/v1.5.2) - 10 Apr 2015 +### [1.5.2](https://github.com/magiconair/properties/tree/v1.5.2) - 10 Apr 2015 * [Issue #3](https://github.com/magiconair/properties/issues/3): Don't print comments in [WriteComment()](http://godoc.org/github.com/magiconair/properties#Properties.WriteComment) if they are all empty * Add clickable links to README -### [1.5.1](https://github.com/magiconair/properties/tags/v1.5.1) - 08 Dec 2014 +### [1.5.1](https://github.com/magiconair/properties/tree/v1.5.1) - 08 Dec 2014 * Added [GetParsedDuration()](http://godoc.org/github.com/magiconair/properties#Properties.GetParsedDuration) and [MustGetParsedDuration()](http://godoc.org/github.com/magiconair/properties#Properties.MustGetParsedDuration) for values specified compatible with [time.ParseDuration()](http://golang.org/pkg/time/#ParseDuration). -### [1.5.0](https://github.com/magiconair/properties/tags/v1.5.0) - 18 Nov 2014 +### [1.5.0](https://github.com/magiconair/properties/tree/v1.5.0) - 18 Nov 2014 * Added support for single and multi-line comments (reading, writing and updating) * The order of keys is now preserved @@ -59,31 +86,31 @@ * Added a [MustSet()](http://godoc.org/github.com/magiconair/properties#Properties.MustSet) method * Migrated test library from launchpad.net/gocheck to [gopkg.in/check.v1](http://gopkg.in/check.v1) -### [1.4.2](https://github.com/magiconair/properties/tags/v1.4.2) - 15 Nov 2014 +### [1.4.2](https://github.com/magiconair/properties/tree/v1.4.2) - 15 Nov 2014 * [Issue #2](https://github.com/magiconair/properties/issues/2): Fixed goroutine leak in parser which created two lexers but cleaned up only one -### [1.4.1](https://github.com/magiconair/properties/tags/v1.4.1) - 13 Nov 2014 +### [1.4.1](https://github.com/magiconair/properties/tree/v1.4.1) - 13 Nov 2014 * [Issue #1](https://github.com/magiconair/properties/issues/1): Fixed bug in Keys() method which returned an empty string -### [1.4.0](https://github.com/magiconair/properties/tags/v1.4.0) - 23 Sep 2014 +### [1.4.0](https://github.com/magiconair/properties/tree/v1.4.0) - 23 Sep 2014 * Added [Keys()](http://godoc.org/github.com/magiconair/properties#Properties.Keys) to get the keys * Added [Filter()](http://godoc.org/github.com/magiconair/properties#Properties.Filter), [FilterRegexp()](http://godoc.org/github.com/magiconair/properties#Properties.FilterRegexp) and [FilterPrefix()](http://godoc.org/github.com/magiconair/properties#Properties.FilterPrefix) to get a subset of the properties -### [1.3.0](https://github.com/magiconair/properties/tags/v1.3.0) - 18 Mar 2014 +### [1.3.0](https://github.com/magiconair/properties/tree/v1.3.0) - 18 Mar 2014 * Added support for time.Duration -* Made MustXXX() failure beha[ior configurable (log.Fatal, panic](https://github.com/magiconair/properties/tags/vior configurable (log.Fatal, panic) - custom) +* Made MustXXX() failure beha[ior configurable (log.Fatal, panic](https://github.com/magiconair/properties/tree/vior configurable (log.Fatal, panic) - custom) * Changed default of MustXXX() failure from panic to log.Fatal -### [1.2.0](https://github.com/magiconair/properties/tags/v1.2.0) - 05 Mar 2014 +### [1.2.0](https://github.com/magiconair/properties/tree/v1.2.0) - 05 Mar 2014 * Added MustGet... functions * Added support for int and uint with range checks on 32 bit platforms -### [1.1.0](https://github.com/magiconair/properties/tags/v1.1.0) - 20 Jan 2014 +### [1.1.0](https://github.com/magiconair/properties/tree/v1.1.0) - 20 Jan 2014 * Renamed from goproperties to properties * Added support for expansion of environment vars in @@ -91,6 +118,6 @@ * Fixed bug where value expressions were not at the start of the string -### [1.0.0](https://github.com/magiconair/properties/tags/v1.0.0) - 7 Jan 2014 +### [1.0.0](https://github.com/magiconair/properties/tree/v1.0.0) - 7 Jan 2014 * Initial release diff --git a/vendor/github.com/magiconair/properties/LICENSE b/vendor/github.com/magiconair/properties/LICENSE index 7eab43b6..b387087c 100644 --- a/vendor/github.com/magiconair/properties/LICENSE +++ b/vendor/github.com/magiconair/properties/LICENSE @@ -1,6 +1,6 @@ goproperties - properties file decoder for Go -Copyright (c) 2013-2014 - Frank Schroeder +Copyright (c) 2013-2018 - Frank Schroeder All rights reserved. diff --git a/vendor/github.com/magiconair/properties/README.md b/vendor/github.com/magiconair/properties/README.md index 71b6a535..2c05f290 100644 --- a/vendor/github.com/magiconair/properties/README.md +++ b/vendor/github.com/magiconair/properties/README.md @@ -1,7 +1,12 @@ -Overview [![Build Status](https://travis-ci.org/magiconair/properties.svg?branch=master)](https://travis-ci.org/magiconair/properties) -======== +[![](https://img.shields.io/github/tag/magiconair/properties.svg?style=flat-square&label=release)](https://github.com/magiconair/properties/releases) +[![Travis CI Status](https://img.shields.io/travis/magiconair/properties.svg?branch=master&style=flat-square&label=travis)](https://travis-ci.org/magiconair/properties) +[![Codeship CI Status](https://img.shields.io/codeship/16aaf660-f615-0135-b8f0-7e33b70920c0/master.svg?label=codeship&style=flat-square)](https://app.codeship.com/projects/274177") +[![License](https://img.shields.io/badge/License-BSD%202--Clause-orange.svg?style=flat-square)](https://raw.githubusercontent.com/magiconair/properties/master/LICENSE) +[![GoDoc](http://img.shields.io/badge/godoc-reference-5272B4.svg?style=flat-square)](http://godoc.org/github.com/magiconair/properties) -#### Current version: 1.7.2 +# Overview + +#### Please run `git pull --tags` to update the tags. See [below](#updated-git-tags) why. properties is a Go library for reading and writing properties files. @@ -27,8 +32,7 @@ details. Read the full documentation on [GoDoc](https://godoc.org/github.com/magiconair/properties) [![GoDoc](https://godoc.org/github.com/magiconair/properties?status.png)](https://godoc.org/github.com/magiconair/properties) -Getting Started ---------------- +## Getting Started ```go import ( @@ -83,18 +87,43 @@ func main() { ``` -Installation and Upgrade ------------------------- +## Installation and Upgrade ``` $ go get -u github.com/magiconair/properties ``` -License -------- +## License 2 clause BSD license. See [LICENSE](https://github.com/magiconair/properties/blob/master/LICENSE) file for details. -ToDo ----- +## ToDo + * Dump contents with passwords and secrets obscured + +## Updated Git tags + +#### 13 Feb 2018 + +I realized that all of the git tags I had pushed before v1.7.5 were lightweight tags +and I've only recently learned that this doesn't play well with `git describe` 😞 + +I have replaced all lightweight tags with signed tags using this script which should +retain the commit date, name and email address. Please run `git pull --tags` to update them. + +Worst case you have to reclone the repo. + +```shell +#!/bin/bash +tag=$1 +echo "Updating $tag" +date=$(git show ${tag}^0 --format=%aD | head -1) +email=$(git show ${tag}^0 --format=%aE | head -1) +name=$(git show ${tag}^0 --format=%aN | head -1) +GIT_COMMITTER_DATE="$date" GIT_COMMITTER_NAME="$name" GIT_COMMITTER_EMAIL="$email" git tag -s -f ${tag} ${tag}^0 -m ${tag} +``` + +I apologize for the inconvenience. + +Frank + diff --git a/vendor/github.com/magiconair/properties/lex.go b/vendor/github.com/magiconair/properties/lex.go index a3cba031..c63fcc60 100644 --- a/vendor/github.com/magiconair/properties/lex.go +++ b/vendor/github.com/magiconair/properties/lex.go @@ -196,9 +196,8 @@ func lexBeforeKey(l *lexer) stateFn { return lexComment case isWhitespace(r): - l.acceptRun(whitespace) l.ignore() - return lexKey + return lexBeforeKey default: l.backup() diff --git a/vendor/github.com/magiconair/properties/load.go b/vendor/github.com/magiconair/properties/load.go index 278cc2ea..9c83fd63 100644 --- a/vendor/github.com/magiconair/properties/load.go +++ b/vendor/github.com/magiconair/properties/load.go @@ -218,7 +218,7 @@ func must(p *Properties, err error) *Properties { // with an empty string. Malformed expressions like "${ENV_VAR" will // be reported as error. func expandName(name string) (string, error) { - return expand(name, make(map[string]bool), "${", "}", make(map[string]string)) + return expand(name, []string{}, "${", "}", make(map[string]string)) } // Interprets a byte buffer either as an ISO-8859-1 or UTF-8 encoded string. diff --git a/vendor/github.com/magiconair/properties/properties.go b/vendor/github.com/magiconair/properties/properties.go index 4f3d5a45..53f5b2ff 100644 --- a/vendor/github.com/magiconair/properties/properties.go +++ b/vendor/github.com/magiconair/properties/properties.go @@ -19,6 +19,8 @@ import ( "unicode/utf8" ) +const maxExpansionDepth = 64 + // ErrorHandlerFunc defines the type of function which handles failures // of the MustXXX() functions. An error handler function must exit // the application after handling the error. @@ -92,7 +94,7 @@ func (p *Properties) Get(key string) (value string, ok bool) { return "", false } - expanded, err := p.expand(v) + expanded, err := p.expand(key, v) // we guarantee that the expanded value is free of // circular references and malformed expressions @@ -511,6 +513,9 @@ func (p *Properties) Set(key, value string) (prev string, ok bool, err error) { if p.DisableExpansion { prev, ok = p.Get(key) p.m[key] = value + if !ok { + p.k = append(p.k, key) + } return prev, ok, nil } @@ -522,7 +527,7 @@ func (p *Properties) Set(key, value string) (prev string, ok bool, err error) { p.m[key] = value // now check for a circular reference - _, err = p.expand(value) + _, err = p.expand(key, value) if err != nil { // revert to the previous state @@ -693,56 +698,65 @@ outer: // check expands all values and returns an error if a circular reference or // a malformed expression was found. func (p *Properties) check() error { - for _, value := range p.m { - if _, err := p.expand(value); err != nil { + for key, value := range p.m { + if _, err := p.expand(key, value); err != nil { return err } } return nil } -func (p *Properties) expand(input string) (string, error) { +func (p *Properties) expand(key, input string) (string, error) { // no pre/postfix -> nothing to expand if p.Prefix == "" && p.Postfix == "" { return input, nil } - return expand(input, make(map[string]bool), p.Prefix, p.Postfix, p.m) + return expand(input, []string{key}, p.Prefix, p.Postfix, p.m) } // expand recursively expands expressions of '(prefix)key(postfix)' to their corresponding values. // The function keeps track of the keys that were already expanded and stops if it // detects a circular reference or a malformed expression of the form '(prefix)key'. -func expand(s string, keys map[string]bool, prefix, postfix string, values map[string]string) (string, error) { - start := strings.Index(s, prefix) - if start == -1 { - return s, nil +func expand(s string, keys []string, prefix, postfix string, values map[string]string) (string, error) { + if len(keys) > maxExpansionDepth { + return "", fmt.Errorf("expansion too deep") } - keyStart := start + len(prefix) - keyLen := strings.Index(s[keyStart:], postfix) - if keyLen == -1 { - return "", fmt.Errorf("malformed expression") + for { + start := strings.Index(s, prefix) + if start == -1 { + return s, nil + } + + keyStart := start + len(prefix) + keyLen := strings.Index(s[keyStart:], postfix) + if keyLen == -1 { + return "", fmt.Errorf("malformed expression") + } + + end := keyStart + keyLen + len(postfix) - 1 + key := s[keyStart : keyStart+keyLen] + + // fmt.Printf("s:%q pp:%q start:%d end:%d keyStart:%d keyLen:%d key:%q\n", s, prefix + "..." + postfix, start, end, keyStart, keyLen, key) + + for _, k := range keys { + if key == k { + return "", fmt.Errorf("circular reference") + } + } + + val, ok := values[key] + if !ok { + val = os.Getenv(key) + } + new_val, err := expand(val, append(keys, key), prefix, postfix, values) + if err != nil { + return "", err + } + s = s[:start] + new_val + s[end+1:] } - - end := keyStart + keyLen + len(postfix) - 1 - key := s[keyStart : keyStart+keyLen] - - // fmt.Printf("s:%q pp:%q start:%d end:%d keyStart:%d keyLen:%d key:%q\n", s, prefix + "..." + postfix, start, end, keyStart, keyLen, key) - - if _, ok := keys[key]; ok { - return "", fmt.Errorf("circular reference") - } - - val, ok := values[key] - if !ok { - val = os.Getenv(key) - } - - // remember that we've seen the key - keys[key] = true - - return expand(s[:start]+val+s[end+1:], keys, prefix, postfix, values) + return s, nil } // encode encodes a UTF-8 string to ISO-8859-1 and escapes some characters. diff --git a/vendor/github.com/miekg/dns/CONTRIBUTORS b/vendor/github.com/miekg/dns/CONTRIBUTORS index f77e8a89..5903779d 100644 --- a/vendor/github.com/miekg/dns/CONTRIBUTORS +++ b/vendor/github.com/miekg/dns/CONTRIBUTORS @@ -7,3 +7,4 @@ Marek Majkowski Peter van Dijk Omri Bahumi Alex Sergeyev +James Hartig diff --git a/vendor/github.com/miekg/dns/Gopkg.lock b/vendor/github.com/miekg/dns/Gopkg.lock new file mode 100644 index 00000000..0c73a644 --- /dev/null +++ b/vendor/github.com/miekg/dns/Gopkg.lock @@ -0,0 +1,21 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = ["ed25519","ed25519/internal/edwards25519"] + revision = "b080dc9a8c480b08e698fb1219160d598526310f" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["bpf","internal/iana","internal/socket","ipv4","ipv6"] + revision = "894f8ed5849b15b810ae41e9590a0d05395bba27" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "c4abc38abaeeeeb9be92455c9c02cae32841122b8982aaa067ef25bb8e86ff9d" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/vendor/github.com/miekg/dns/Gopkg.toml b/vendor/github.com/miekg/dns/Gopkg.toml new file mode 100644 index 00000000..2f655b2c --- /dev/null +++ b/vendor/github.com/miekg/dns/Gopkg.toml @@ -0,0 +1,26 @@ + +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + + +[[constraint]] + branch = "master" + name = "golang.org/x/crypto" diff --git a/vendor/github.com/miekg/dns/Makefile.fuzz b/vendor/github.com/miekg/dns/Makefile.fuzz new file mode 100644 index 00000000..dc158c4a --- /dev/null +++ b/vendor/github.com/miekg/dns/Makefile.fuzz @@ -0,0 +1,33 @@ +# Makefile for fuzzing +# +# Use go-fuzz and needs the tools installed. +# See https://blog.cloudflare.com/dns-parser-meet-go-fuzzer/ +# +# Installing go-fuzz: +# $ make -f Makefile.fuzz get +# Installs: +# * github.com/dvyukov/go-fuzz/go-fuzz +# * get github.com/dvyukov/go-fuzz/go-fuzz-build + +all: build + +.PHONY: build +build: + go-fuzz-build -tags fuzz github.com/miekg/dns + +.PHONY: build-newrr +build-newrr: + go-fuzz-build -func FuzzNewRR -tags fuzz github.com/miekg/dns + +.PHONY: fuzz +fuzz: + go-fuzz -bin=dns-fuzz.zip -workdir=fuzz + +.PHONY: get +get: + go get github.com/dvyukov/go-fuzz/go-fuzz + go get github.com/dvyukov/go-fuzz/go-fuzz-build + +.PHONY: clean +clean: + rm *-fuzz.zip diff --git a/vendor/github.com/miekg/dns/Makefile.release b/vendor/github.com/miekg/dns/Makefile.release new file mode 100644 index 00000000..8fb748e8 --- /dev/null +++ b/vendor/github.com/miekg/dns/Makefile.release @@ -0,0 +1,52 @@ +# Makefile for releasing. +# +# The release is controlled from version.go. The version found there is +# used to tag the git repo, we're not building any artifects so there is nothing +# to upload to github. +# +# * Up the version in version.go +# * Run: make -f Makefile.release release +# * will *commit* your change with 'Release $VERSION' +# * push to github +# + +define GO +//+build ignore + +package main + +import ( + "fmt" + + "github.com/miekg/dns" +) + +func main() { + fmt.Println(dns.Version.String()) +} +endef + +$(file > version_release.go,$(GO)) +VERSION:=$(shell go run version_release.go) +TAG="v$(VERSION)" + +all: + @echo Use the \'release\' target to start a release $(VERSION) + rm -f version_release.go + +.PHONY: release +release: commit push + @echo Released $(VERSION) + rm -f version_release.go + +.PHONY: commit +commit: + @echo Committing release $(VERSION) + git commit -am"Release $(VERSION)" + git tag $(TAG) + +.PHONY: push +push: + @echo Pushing release $(VERSION) to master + git push --tags + git push diff --git a/vendor/github.com/miekg/dns/README.md b/vendor/github.com/miekg/dns/README.md index cd5c1948..1ad23c75 100644 --- a/vendor/github.com/miekg/dns/README.md +++ b/vendor/github.com/miekg/dns/README.md @@ -1,19 +1,19 @@ [![Build Status](https://travis-ci.org/miekg/dns.svg?branch=master)](https://travis-ci.org/miekg/dns) +[![Code Coverage](https://img.shields.io/codecov/c/github/miekg/dns/master.svg)](https://codecov.io/github/miekg/dns?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/miekg/dns)](https://goreportcard.com/report/miekg/dns) [![](https://godoc.org/github.com/miekg/dns?status.svg)](https://godoc.org/github.com/miekg/dns) # Alternative (more granular) approach to a DNS library > Less is more. -Complete and usable DNS library. All widely used Resource Records are -supported, including the DNSSEC types. It follows a lean and mean philosophy. -If there is stuff you should know as a DNS programmer there isn't a convenience -function for it. Server side and client side programming is supported, i.e. you -can build servers and resolvers with it. +Complete and usable DNS library. All widely used Resource Records are supported, including the +DNSSEC types. It follows a lean and mean philosophy. If there is stuff you should know as a DNS +programmer there isn't a convenience function for it. Server side and client side programming is +supported, i.e. you can build servers and resolvers with it. -We try to keep the "master" branch as sane as possible and at the bleeding edge -of standards, avoiding breaking changes wherever reasonable. We support the last -two versions of Go, currently: 1.6 and 1.7. +We try to keep the "master" branch as sane as possible and at the bleeding edge of standards, +avoiding breaking changes wherever reasonable. We support the last two versions of Go. # Goals @@ -55,9 +55,15 @@ A not-so-up-to-date-list-that-may-be-actually-current: * https://github.com/mehrdadrad/mylg * https://github.com/bamarni/dockness * https://github.com/fffaraz/microdns -* http://quilt.io +* http://kelda.io * https://github.com/ipdcode/hades (JD.COM) * https://github.com/StackExchange/dnscontrol/ +* https://www.dnsperf.com/ +* https://dnssectest.net/ +* https://dns.apebits.com +* https://github.com/oif/apex +* https://github.com/jedisct1/dnscrypt-proxy +* https://github.com/jedisct1/rpdns Send pull request if you want to be listed here. @@ -70,7 +76,7 @@ Send pull request if you want to be listed here. * Parsing RRs ~ 100K RR/s, that's 5M records in about 50 seconds; * Server side programming (mimicking the net/http package); * Client side programming; -* DNSSEC: signing, validating and key generation for DSA, RSA and ECDSA; +* DNSSEC: signing, validating and key generation for DSA, RSA, ECDSA and Ed25519; * EDNS0, NSID, Cookies; * AXFR/IXFR; * TSIG, SIG(0); @@ -84,8 +90,8 @@ Miek Gieben - 2010-2012 - # Building -Building is done with the `go` tool. If you have setup your GOPATH -correctly, the following should work: +Building is done with the `go` tool. If you have setup your GOPATH correctly, the following should +work: go get github.com/miekg/dns go build github.com/miekg/dns @@ -146,11 +152,13 @@ Example programs can be found in the `github.com/miekg/exdns` repository. * 6975 - Algorithm Understanding in DNSSEC * 7043 - EUI48/EUI64 records * 7314 - DNS (EDNS) EXPIRE Option +* 7477 - CSYNC RR * 7828 - edns-tcp-keepalive EDNS0 Option * 7553 - URI record -* 7858 - DNS over TLS: Initiation and Performance Considerations (draft) +* 7858 - DNS over TLS: Initiation and Performance Considerations +* 7871 - EDNS0 Client Subnet * 7873 - Domain Name System (DNS) Cookies (draft-ietf-dnsop-cookies) -* xxxx - EDNS0 DNS Update Lease (draft) +* 8080 - EdDSA for DNSSEC ## Loosely based upon diff --git a/vendor/github.com/miekg/dns/client.go b/vendor/github.com/miekg/dns/client.go index 301dab9c..282565af 100644 --- a/vendor/github.com/miekg/dns/client.go +++ b/vendor/github.com/miekg/dns/client.go @@ -4,10 +4,12 @@ package dns import ( "bytes" + "context" "crypto/tls" "encoding/binary" "io" "net" + "strings" "time" ) @@ -18,22 +20,24 @@ const tcpIdleTimeout time.Duration = 8 * time.Second type Conn struct { net.Conn // a net.Conn holding the connection UDPSize uint16 // minimum receive buffer for UDP messages - TsigSecret map[string]string // secret(s) for Tsig map[], zonename must be fully qualified - rtt time.Duration - t time.Time + TsigSecret map[string]string // secret(s) for Tsig map[], zonename must be in canonical form (lowercase, fqdn, see RFC 4034 Section 6.2) tsigRequestMAC string } // A Client defines parameters for a DNS client. type Client struct { - Net string // if "tcp" or "tcp-tls" (DNS over TLS) a TCP query will be initiated, otherwise an UDP one (default is "" for UDP) - UDPSize uint16 // minimum receive buffer for UDP messages - TLSConfig *tls.Config // TLS connection configuration - Timeout time.Duration // a cumulative timeout for dial, write and read, defaults to 0 (disabled) - overrides DialTimeout, ReadTimeout and WriteTimeout when non-zero - DialTimeout time.Duration // net.DialTimeout, defaults to 2 seconds - overridden by Timeout when that value is non-zero + Net string // if "tcp" or "tcp-tls" (DNS over TLS) a TCP query will be initiated, otherwise an UDP one (default is "" for UDP) + UDPSize uint16 // minimum receive buffer for UDP messages + TLSConfig *tls.Config // TLS connection configuration + Dialer *net.Dialer // a net.Dialer used to set local address, timeouts and more + // Timeout is a cumulative timeout for dial, write and read, defaults to 0 (disabled) - overrides DialTimeout, ReadTimeout, + // WriteTimeout when non-zero. Can be overridden with net.Dialer.Timeout (see Client.ExchangeWithDialer and + // Client.Dialer) or context.Context.Deadline (see the deprecated ExchangeContext) + Timeout time.Duration + DialTimeout time.Duration // net.DialTimeout, defaults to 2 seconds, or net.Dialer.Timeout if expiring earlier - overridden by Timeout when that value is non-zero ReadTimeout time.Duration // net.Conn.SetReadTimeout value for connections, defaults to 2 seconds - overridden by Timeout when that value is non-zero WriteTimeout time.Duration // net.Conn.SetWriteTimeout value for connections, defaults to 2 seconds - overridden by Timeout when that value is non-zero - TsigSecret map[string]string // secret(s) for Tsig map[], zonename must be fully qualified + TsigSecret map[string]string // secret(s) for Tsig map[], zonename must be in canonical form (lowercase, fqdn, see RFC 4034 Section 6.2) SingleInflight bool // if true suppress multiple outstanding queries for the same Qname, Qtype and Qclass group singleflight } @@ -43,93 +47,11 @@ type Client struct { // will it fall back to TCP in case of truncation. // See client.Exchange for more information on setting larger buffer sizes. func Exchange(m *Msg, a string) (r *Msg, err error) { - var co *Conn - co, err = DialTimeout("udp", a, dnsTimeout) - if err != nil { - return nil, err - } - - defer co.Close() - - opt := m.IsEdns0() - // If EDNS0 is used use that for size. - if opt != nil && opt.UDPSize() >= MinMsgSize { - co.UDPSize = opt.UDPSize() - } - - co.SetWriteDeadline(time.Now().Add(dnsTimeout)) - if err = co.WriteMsg(m); err != nil { - return nil, err - } - - co.SetReadDeadline(time.Now().Add(dnsTimeout)) - r, err = co.ReadMsg() - if err == nil && r.Id != m.Id { - err = ErrId - } + client := Client{Net: "udp"} + r, _, err = client.Exchange(m, a) return r, err } -// ExchangeConn performs a synchronous query. It sends the message m via the connection -// c and waits for a reply. The connection c is not closed by ExchangeConn. -// This function is going away, but can easily be mimicked: -// -// co := &dns.Conn{Conn: c} // c is your net.Conn -// co.WriteMsg(m) -// in, _ := co.ReadMsg() -// co.Close() -// -func ExchangeConn(c net.Conn, m *Msg) (r *Msg, err error) { - println("dns: this function is deprecated") - co := new(Conn) - co.Conn = c - if err = co.WriteMsg(m); err != nil { - return nil, err - } - r, err = co.ReadMsg() - if err == nil && r.Id != m.Id { - err = ErrId - } - return r, err -} - -// Exchange performs a synchronous query. It sends the message m to the address -// contained in a and waits for a reply. Basic use pattern with a *dns.Client: -// -// c := new(dns.Client) -// in, rtt, err := c.Exchange(message, "127.0.0.1:53") -// -// Exchange does not retry a failed query, nor will it fall back to TCP in -// case of truncation. -// It is up to the caller to create a message that allows for larger responses to be -// returned. Specifically this means adding an EDNS0 OPT RR that will advertise a larger -// buffer, see SetEdns0. Messages without an OPT RR will fallback to the historic limit -// of 512 bytes. -func (c *Client) Exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err error) { - if !c.SingleInflight { - return c.exchange(m, a) - } - // This adds a bunch of garbage, TODO(miek). - t := "nop" - if t1, ok := TypeToString[m.Question[0].Qtype]; ok { - t = t1 - } - cl := "nop" - if cl1, ok := ClassToString[m.Question[0].Qclass]; ok { - cl = cl1 - } - r, rtt, err, shared := c.group.Do(m.Question[0].Name+t+cl, func() (*Msg, time.Duration, error) { - return c.exchange(m, a) - }) - if r != nil && shared { - r = r.Copy() - } - if err != nil { - return r, rtt, err - } - return r, rtt, nil -} - func (c *Client) dialTimeout() time.Duration { if c.Timeout != 0 { return c.Timeout @@ -154,37 +76,88 @@ func (c *Client) writeTimeout() time.Duration { return dnsTimeout } -func (c *Client) exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err error) { - var co *Conn +// Dial connects to the address on the named network. +func (c *Client) Dial(address string) (conn *Conn, err error) { + // create a new dialer with the appropriate timeout + var d net.Dialer + if c.Dialer == nil { + d = net.Dialer{} + } else { + d = net.Dialer(*c.Dialer) + } + d.Timeout = c.getTimeoutForRequest(c.writeTimeout()) + network := "udp" - tls := false + useTLS := false switch c.Net { case "tcp-tls": network = "tcp" - tls = true + useTLS = true case "tcp4-tls": network = "tcp4" - tls = true + useTLS = true case "tcp6-tls": network = "tcp6" - tls = true + useTLS = true default: if c.Net != "" { network = c.Net } } - var deadline time.Time - if c.Timeout != 0 { - deadline = time.Now().Add(c.Timeout) + conn = new(Conn) + if useTLS { + conn.Conn, err = tls.DialWithDialer(&d, network, address, c.TLSConfig) + } else { + conn.Conn, err = d.Dial(network, address) + } + if err != nil { + return nil, err + } + return conn, nil +} + +// Exchange performs a synchronous query. It sends the message m to the address +// contained in a and waits for a reply. Basic use pattern with a *dns.Client: +// +// c := new(dns.Client) +// in, rtt, err := c.Exchange(message, "127.0.0.1:53") +// +// Exchange does not retry a failed query, nor will it fall back to TCP in +// case of truncation. +// It is up to the caller to create a message that allows for larger responses to be +// returned. Specifically this means adding an EDNS0 OPT RR that will advertise a larger +// buffer, see SetEdns0. Messages without an OPT RR will fallback to the historic limit +// of 512 bytes +// To specify a local address or a timeout, the caller has to set the `Client.Dialer` +// attribute appropriately +func (c *Client) Exchange(m *Msg, address string) (r *Msg, rtt time.Duration, err error) { + if !c.SingleInflight { + return c.exchange(m, address) } - if tls { - co, err = DialTimeoutWithTLS(network, a, c.TLSConfig, c.dialTimeout()) - } else { - co, err = DialTimeout(network, a, c.dialTimeout()) + t := "nop" + if t1, ok := TypeToString[m.Question[0].Qtype]; ok { + t = t1 } + cl := "nop" + if cl1, ok := ClassToString[m.Question[0].Qclass]; ok { + cl = cl1 + } + r, rtt, err, shared := c.group.Do(m.Question[0].Name+t+cl, func() (*Msg, time.Duration, error) { + return c.exchange(m, address) + }) + if r != nil && shared { + r = r.Copy() + } + return r, rtt, err +} + +func (c *Client) exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err error) { + var co *Conn + + co, err = c.Dial(a) if err != nil { return nil, 0, err @@ -202,22 +175,27 @@ func (c *Client) exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err erro } co.TsigSecret = c.TsigSecret - co.SetWriteDeadline(deadlineOrTimeout(deadline, c.writeTimeout())) + t := time.Now() + // write with the appropriate write timeout + co.SetWriteDeadline(t.Add(c.getTimeoutForRequest(c.writeTimeout()))) if err = co.WriteMsg(m); err != nil { return nil, 0, err } - co.SetReadDeadline(deadlineOrTimeout(deadline, c.readTimeout())) + co.SetReadDeadline(time.Now().Add(c.getTimeoutForRequest(c.readTimeout()))) r, err = co.ReadMsg() if err == nil && r.Id != m.Id { err = ErrId } - return r, co.rtt, err + rtt = time.Since(t) + return r, rtt, err } // ReadMsg reads a message from the connection co. -// If the received message contains a TSIG record the transaction -// signature is verified. +// If the received message contains a TSIG record the transaction signature +// is verified. This method always tries to return the message, however if an +// error is returned there are no guarantees that the returned message is a +// valid representation of the packet read. func (co *Conn) ReadMsg() (*Msg, error) { p, err := co.ReadMsgHeader(nil) if err != nil { @@ -226,13 +204,10 @@ func (co *Conn) ReadMsg() (*Msg, error) { m := new(Msg) if err := m.Unpack(p); err != nil { - // If ErrTruncated was returned, we still want to allow the user to use + // If an error was returned, we still want to allow the user to use // the message, but naively they can just check err if they don't want - // to use a truncated message - if err == ErrTruncated { - return m, err - } - return nil, err + // to use an erroneous message + return m, err } if t := m.IsTsig(); t != nil { if _, ok := co.TsigSecret[t.Hdr.Name]; !ok { @@ -265,7 +240,6 @@ func (co *Conn) ReadMsgHeader(hdr *Header) ([]byte, error) { } p = make([]byte, l) n, err = tcpRead(r, p) - co.rtt = time.Since(co.t) default: if co.UDPSize > MinMsgSize { p = make([]byte, co.UDPSize) @@ -273,7 +247,6 @@ func (co *Conn) ReadMsgHeader(hdr *Header) ([]byte, error) { p = make([]byte, MinMsgSize) } n, err = co.Read(p) - co.rtt = time.Since(co.t) } if err != nil { @@ -301,7 +274,7 @@ func tcpMsgLen(t io.Reader) (int, error) { return 0, err } - // As seen with my local router/switch, retursn 1 byte on the above read, + // As seen with my local router/switch, returns 1 byte on the above read, // resulting a a ShortRead. Just write it out (instead of loop) and read the // other byte. if n == 1 { @@ -386,7 +359,6 @@ func (co *Conn) WriteMsg(m *Msg) (err error) { if err != nil { return err } - co.t = time.Now() if _, err = co.Write(out); err != nil { return err } @@ -416,6 +388,24 @@ func (co *Conn) Write(p []byte) (n int, err error) { return n, err } +// Return the appropriate timeout for a specific request +func (c *Client) getTimeoutForRequest(timeout time.Duration) time.Duration { + var requestTimeout time.Duration + if c.Timeout != 0 { + requestTimeout = c.Timeout + } else { + requestTimeout = timeout + } + // net.Dialer.Timeout has priority if smaller than the timeouts computed so + // far + if c.Dialer != nil && c.Dialer.Timeout != 0 { + if c.Dialer.Timeout < requestTimeout { + requestTimeout = c.Dialer.Timeout + } + } + return requestTimeout +} + // Dial connects to the address on the named network. func Dial(network, address string) (conn *Conn, err error) { conn = new(Conn) @@ -426,10 +416,43 @@ func Dial(network, address string) (conn *Conn, err error) { return conn, nil } +// ExchangeContext performs a synchronous UDP query, like Exchange. It +// additionally obeys deadlines from the passed Context. +func ExchangeContext(ctx context.Context, m *Msg, a string) (r *Msg, err error) { + client := Client{Net: "udp"} + r, _, err = client.ExchangeContext(ctx, m, a) + // ignorint rtt to leave the original ExchangeContext API unchanged, but + // this function will go away + return r, err +} + +// ExchangeConn performs a synchronous query. It sends the message m via the connection +// c and waits for a reply. The connection c is not closed by ExchangeConn. +// This function is going away, but can easily be mimicked: +// +// co := &dns.Conn{Conn: c} // c is your net.Conn +// co.WriteMsg(m) +// in, _ := co.ReadMsg() +// co.Close() +// +func ExchangeConn(c net.Conn, m *Msg) (r *Msg, err error) { + println("dns: ExchangeConn: this function is deprecated") + co := new(Conn) + co.Conn = c + if err = co.WriteMsg(m); err != nil { + return nil, err + } + r, err = co.ReadMsg() + if err == nil && r.Id != m.Id { + err = ErrId + } + return r, err +} + // DialTimeout acts like Dial but takes a timeout. func DialTimeout(network, address string, timeout time.Duration) (conn *Conn, err error) { - conn = new(Conn) - conn.Conn, err = net.DialTimeout(network, address, timeout) + client := Client{Net: network, Dialer: &net.Dialer{Timeout: timeout}} + conn, err = client.Dial(address) if err != nil { return nil, err } @@ -438,8 +461,12 @@ func DialTimeout(network, address string, timeout time.Duration) (conn *Conn, er // DialWithTLS connects to the address on the named network with TLS. func DialWithTLS(network, address string, tlsConfig *tls.Config) (conn *Conn, err error) { - conn = new(Conn) - conn.Conn, err = tls.Dial(network, address, tlsConfig) + if !strings.HasSuffix(network, "-tls") { + network += "-tls" + } + client := Client{Net: network, TLSConfig: tlsConfig} + conn, err = client.Dial(address) + if err != nil { return nil, err } @@ -448,20 +475,29 @@ func DialWithTLS(network, address string, tlsConfig *tls.Config) (conn *Conn, er // DialTimeoutWithTLS acts like DialWithTLS but takes a timeout. func DialTimeoutWithTLS(network, address string, tlsConfig *tls.Config, timeout time.Duration) (conn *Conn, err error) { - var dialer net.Dialer - dialer.Timeout = timeout - - conn = new(Conn) - conn.Conn, err = tls.DialWithDialer(&dialer, network, address, tlsConfig) + if !strings.HasSuffix(network, "-tls") { + network += "-tls" + } + client := Client{Net: network, Dialer: &net.Dialer{Timeout: timeout}, TLSConfig: tlsConfig} + conn, err = client.Dial(address) if err != nil { return nil, err } return conn, nil } -func deadlineOrTimeout(deadline time.Time, timeout time.Duration) time.Time { - if deadline.IsZero() { - return time.Now().Add(timeout) +// ExchangeContext acts like Exchange, but honors the deadline on the provided +// context, if present. If there is both a context deadline and a configured +// timeout on the client, the earliest of the two takes effect. +func (c *Client) ExchangeContext(ctx context.Context, m *Msg, a string) (r *Msg, rtt time.Duration, err error) { + var timeout time.Duration + if deadline, ok := ctx.Deadline(); !ok { + timeout = 0 + } else { + timeout = deadline.Sub(time.Now()) } - return deadline + // not passing the context to the underlying calls, as the API does not support + // context. For timeouts you should set up Client.Dialer and call Client.Exchange. + c.Dialer = &net.Dialer{Timeout: timeout} + return c.Exchange(m, a) } diff --git a/vendor/github.com/miekg/dns/clientconfig.go b/vendor/github.com/miekg/dns/clientconfig.go index 0a1f5a92..f13cfa30 100644 --- a/vendor/github.com/miekg/dns/clientconfig.go +++ b/vendor/github.com/miekg/dns/clientconfig.go @@ -2,6 +2,7 @@ package dns import ( "bufio" + "io" "os" "strconv" "strings" @@ -25,8 +26,13 @@ func ClientConfigFromFile(resolvconf string) (*ClientConfig, error) { return nil, err } defer file.Close() + return ClientConfigFromReader(file) +} + +// ClientConfigFromReader works like ClientConfigFromFile but takes an io.Reader as argument +func ClientConfigFromReader(resolvconf io.Reader) (*ClientConfig, error) { c := new(ClientConfig) - scanner := bufio.NewScanner(file) + scanner := bufio.NewScanner(resolvconf) c.Servers = make([]string, 0) c.Search = make([]string, 0) c.Port = "53" @@ -73,8 +79,10 @@ func ClientConfigFromFile(resolvconf string) (*ClientConfig, error) { switch { case len(s) >= 6 && s[:6] == "ndots:": n, _ := strconv.Atoi(s[6:]) - if n < 1 { - n = 1 + if n < 0 { + n = 0 + } else if n > 15 { + n = 15 } c.Ndots = n case len(s) >= 8 && s[:8] == "timeout:": @@ -83,7 +91,7 @@ func ClientConfigFromFile(resolvconf string) (*ClientConfig, error) { n = 1 } c.Timeout = n - case len(s) >= 8 && s[:9] == "attempts:": + case len(s) >= 9 && s[:9] == "attempts:": n, _ := strconv.Atoi(s[9:]) if n < 1 { n = 1 diff --git a/vendor/github.com/miekg/dns/compress_generate.go b/vendor/github.com/miekg/dns/compress_generate.go index 1a301e9f..87fb36f6 100644 --- a/vendor/github.com/miekg/dns/compress_generate.go +++ b/vendor/github.com/miekg/dns/compress_generate.go @@ -18,8 +18,7 @@ import ( ) var packageHdr = ` -// *** DO NOT MODIFY *** -// AUTOGENERATED BY go generate from compress_generate.go +// Code generated by "go run compress_generate.go"; DO NOT EDIT. package dns @@ -51,8 +50,9 @@ func main() { fatalIfErr(err) scope := pkg.Scope() - domainTypes := map[string]bool{} // Types that have a domain name in them (either comressible or not). - cdomainTypes := map[string]bool{} // Types that have a compressible domain name in them (subset of domainType) + var domainTypes []string // Types that have a domain name in them (either compressible or not). + var cdomainTypes []string // Types that have a compressible domain name in them (subset of domainType) +Names: for _, name := range scope.Names() { o := scope.Lookup(name) if o == nil || !o.Exported() { @@ -73,21 +73,25 @@ func main() { for i := 1; i < st.NumFields(); i++ { if _, ok := st.Field(i).Type().(*types.Slice); ok { if st.Tag(i) == `dns:"domain-name"` { - domainTypes[o.Name()] = true + domainTypes = append(domainTypes, o.Name()) + continue Names } if st.Tag(i) == `dns:"cdomain-name"` { - cdomainTypes[o.Name()] = true - domainTypes[o.Name()] = true + cdomainTypes = append(cdomainTypes, o.Name()) + domainTypes = append(domainTypes, o.Name()) + continue Names } continue } switch { case st.Tag(i) == `dns:"domain-name"`: - domainTypes[o.Name()] = true + domainTypes = append(domainTypes, o.Name()) + continue Names case st.Tag(i) == `dns:"cdomain-name"`: - cdomainTypes[o.Name()] = true - domainTypes[o.Name()] = true + cdomainTypes = append(cdomainTypes, o.Name()) + domainTypes = append(domainTypes, o.Name()) + continue Names } } } @@ -99,7 +103,7 @@ func main() { fmt.Fprint(b, "func compressionLenHelperType(c map[string]int, r RR) {\n") fmt.Fprint(b, "switch x := r.(type) {\n") - for name, _ := range domainTypes { + for _, name := range domainTypes { o := scope.Lookup(name) st, _ := getTypeStruct(o.Type(), scope) @@ -135,7 +139,7 @@ func main() { fmt.Fprint(b, "func compressionLenSearchType(c map[string]int, r RR) (int, bool) {\n") fmt.Fprint(b, "switch x := r.(type) {\n") - for name, _ := range cdomainTypes { + for _, name := range cdomainTypes { o := scope.Lookup(name) st, _ := getTypeStruct(o.Type(), scope) diff --git a/vendor/github.com/miekg/dns/defaults.go b/vendor/github.com/miekg/dns/defaults.go index 3308ec83..14e18b0b 100644 --- a/vendor/github.com/miekg/dns/defaults.go +++ b/vendor/github.com/miekg/dns/defaults.go @@ -13,9 +13,12 @@ const hexDigit = "0123456789abcdef" // SetReply creates a reply message from a request message. func (dns *Msg) SetReply(request *Msg) *Msg { dns.Id = request.Id - dns.RecursionDesired = request.RecursionDesired // Copy rd bit dns.Response = true - dns.Opcode = OpcodeQuery + dns.Opcode = request.Opcode + if dns.Opcode == OpcodeQuery { + dns.RecursionDesired = request.RecursionDesired // Copy rd bit + dns.CheckingDisabled = request.CheckingDisabled // Copy cd bit + } dns.Rcode = RcodeSuccess if len(request.Question) > 0 { dns.Question = make([]Question, 1) @@ -270,8 +273,11 @@ func (t Type) String() string { // String returns the string representation for the class c. func (c Class) String() string { - if c1, ok := ClassToString[uint16(c)]; ok { - return c1 + if s, ok := ClassToString[uint16(c)]; ok { + // Only emit mnemonics when they are unambiguous, specically ANY is in both. + if _, ok := StringToType[s]; !ok { + return s + } } return "CLASS" + strconv.Itoa(int(c)) } diff --git a/vendor/github.com/miekg/dns/dns.go b/vendor/github.com/miekg/dns/dns.go index b3292287..5133eac7 100644 --- a/vendor/github.com/miekg/dns/dns.go +++ b/vendor/github.com/miekg/dns/dns.go @@ -6,9 +6,12 @@ const ( year68 = 1 << 31 // For RFC1982 (Serial Arithmetic) calculations in 32 bits. defaultTtl = 3600 // Default internal TTL. - DefaultMsgSize = 4096 // DefaultMsgSize is the standard default for messages larger than 512 bytes. - MinMsgSize = 512 // MinMsgSize is the minimal size of a DNS packet. - MaxMsgSize = 65535 // MaxMsgSize is the largest possible DNS packet. + // DefaultMsgSize is the standard default for messages larger than 512 bytes. + DefaultMsgSize = 4096 + // MinMsgSize is the minimal size of a DNS packet. + MinMsgSize = 512 + // MaxMsgSize is the largest possible DNS packet. + MaxMsgSize = 65535 ) // Error represents a DNS error. diff --git a/vendor/github.com/miekg/dns/dnssec.go b/vendor/github.com/miekg/dns/dnssec.go index 3bd55388..ac9fdd45 100644 --- a/vendor/github.com/miekg/dns/dnssec.go +++ b/vendor/github.com/miekg/dns/dnssec.go @@ -19,6 +19,8 @@ import ( "sort" "strings" "time" + + "golang.org/x/crypto/ed25519" ) // DNSSEC encryption algorithm codes. @@ -38,6 +40,8 @@ const ( ECCGOST ECDSAP256SHA256 ECDSAP384SHA384 + ED25519 + ED448 INDIRECT uint8 = 252 PRIVATEDNS uint8 = 253 // Private (experimental keys) PRIVATEOID uint8 = 254 @@ -56,6 +60,8 @@ var AlgorithmToString = map[uint8]string{ ECCGOST: "ECC-GOST", ECDSAP256SHA256: "ECDSAP256SHA256", ECDSAP384SHA384: "ECDSAP384SHA384", + ED25519: "ED25519", + ED448: "ED448", INDIRECT: "INDIRECT", PRIVATEDNS: "PRIVATEDNS", PRIVATEOID: "PRIVATEOID", @@ -73,6 +79,7 @@ var AlgorithmToHash = map[uint8]crypto.Hash{ ECDSAP256SHA256: crypto.SHA256, ECDSAP384SHA384: crypto.SHA384, RSASHA512: crypto.SHA512, + ED25519: crypto.Hash(0), } // DNSSEC hashing algorithm codes. @@ -301,17 +308,33 @@ func (rr *RRSIG) Sign(k crypto.Signer, rrset []RR) error { return ErrAlg } - h := hash.New() - h.Write(signdata) - h.Write(wire) + switch rr.Algorithm { + case ED25519: + // ed25519 signs the raw message and performs hashing internally. + // All other supported signature schemes operate over the pre-hashed + // message, and thus ed25519 must be handled separately here. + // + // The raw message is passed directly into sign and crypto.Hash(0) is + // used to signal to the crypto.Signer that the data has not been hashed. + signature, err := sign(k, append(signdata, wire...), crypto.Hash(0), rr.Algorithm) + if err != nil { + return err + } - signature, err := sign(k, h.Sum(nil), hash, rr.Algorithm) - if err != nil { - return err + rr.Signature = toBase64(signature) + default: + h := hash.New() + h.Write(signdata) + h.Write(wire) + + signature, err := sign(k, h.Sum(nil), hash, rr.Algorithm) + if err != nil { + return err + } + + rr.Signature = toBase64(signature) } - rr.Signature = toBase64(signature) - return nil } @@ -352,6 +375,9 @@ func sign(k crypto.Signer, hashed []byte, hash crypto.Hash, alg uint8) ([]byte, // signature = append(signature, intToBytes(r1, 20)...) // signature = append(signature, intToBytes(s1, 20)...) // rr.Signature = signature + + case ED25519: + return signature, nil } return nil, ErrAlg @@ -456,6 +482,17 @@ func (rr *RRSIG) Verify(k *DNSKEY, rrset []RR) error { } return ErrSig + case ED25519: + pubkey := k.publicKeyED25519() + if pubkey == nil { + return ErrKey + } + + if ed25519.Verify(pubkey, append(signeddata, wire...), sigbuf) { + return nil + } + return ErrSig + default: return ErrAlg } @@ -578,6 +615,17 @@ func (k *DNSKEY) publicKeyDSA() *dsa.PublicKey { return pubkey } +func (k *DNSKEY) publicKeyED25519() ed25519.PublicKey { + keybuf, err := fromBase64([]byte(k.PublicKey)) + if err != nil { + return nil + } + if len(keybuf) != ed25519.PublicKeySize { + return nil + } + return keybuf +} + type wireSlice [][]byte func (p wireSlice) Len() int { return len(p) } @@ -615,6 +663,10 @@ func rawSignatureData(rrset []RR, s *RRSIG) (buf []byte, err error) { switch x := r1.(type) { case *NS: x.Ns = strings.ToLower(x.Ns) + case *MD: + x.Md = strings.ToLower(x.Md) + case *MF: + x.Mf = strings.ToLower(x.Mf) case *CNAME: x.Target = strings.ToLower(x.Target) case *SOA: @@ -633,6 +685,18 @@ func rawSignatureData(rrset []RR, s *RRSIG) (buf []byte, err error) { x.Email = strings.ToLower(x.Email) case *MX: x.Mx = strings.ToLower(x.Mx) + case *RP: + x.Mbox = strings.ToLower(x.Mbox) + x.Txt = strings.ToLower(x.Txt) + case *AFSDB: + x.Hostname = strings.ToLower(x.Hostname) + case *RT: + x.Host = strings.ToLower(x.Host) + case *SIG: + x.SignerName = strings.ToLower(x.SignerName) + case *PX: + x.Map822 = strings.ToLower(x.Map822) + x.Mapx400 = strings.ToLower(x.Mapx400) case *NAPTR: x.Replacement = strings.ToLower(x.Replacement) case *KX: diff --git a/vendor/github.com/miekg/dns/dnssec_keygen.go b/vendor/github.com/miekg/dns/dnssec_keygen.go index 5e4b7741..33e913ac 100644 --- a/vendor/github.com/miekg/dns/dnssec_keygen.go +++ b/vendor/github.com/miekg/dns/dnssec_keygen.go @@ -8,6 +8,8 @@ import ( "crypto/rand" "crypto/rsa" "math/big" + + "golang.org/x/crypto/ed25519" ) // Generate generates a DNSKEY of the given bit size. @@ -38,6 +40,10 @@ func (k *DNSKEY) Generate(bits int) (crypto.PrivateKey, error) { if bits != 384 { return nil, ErrKeySize } + case ED25519: + if bits != 256 { + return nil, ErrKeySize + } } switch k.Algorithm { @@ -75,6 +81,13 @@ func (k *DNSKEY) Generate(bits int) (crypto.PrivateKey, error) { } k.setPublicKeyECDSA(priv.PublicKey.X, priv.PublicKey.Y) return priv, nil + case ED25519: + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + k.setPublicKeyED25519(pub) + return priv, nil default: return nil, ErrAlg } @@ -117,6 +130,15 @@ func (k *DNSKEY) setPublicKeyDSA(_Q, _P, _G, _Y *big.Int) bool { return true } +// Set the public key for Ed25519 +func (k *DNSKEY) setPublicKeyED25519(_K ed25519.PublicKey) bool { + if _K == nil { + return false + } + k.PublicKey = toBase64(_K) + return true +} + // Set the public key (the values E and N) for RSA // RFC 3110: Section 2. RSA Public KEY Resource Records func exponentToBuf(_E int) []byte { diff --git a/vendor/github.com/miekg/dns/dnssec_keyscan.go b/vendor/github.com/miekg/dns/dnssec_keyscan.go index 4f8d830b..e2d9d8f9 100644 --- a/vendor/github.com/miekg/dns/dnssec_keyscan.go +++ b/vendor/github.com/miekg/dns/dnssec_keyscan.go @@ -1,6 +1,7 @@ package dns import ( + "bytes" "crypto" "crypto/dsa" "crypto/ecdsa" @@ -9,6 +10,8 @@ import ( "math/big" "strconv" "strings" + + "golang.org/x/crypto/ed25519" ) // NewPrivateKey returns a PrivateKey by parsing the string s. @@ -86,6 +89,8 @@ func (k *DNSKEY) ReadPrivateKey(q io.Reader, file string) (crypto.PrivateKey, er } priv.PublicKey = *pub return priv, nil + case ED25519: + return readPrivateKeyED25519(m) default: return nil, ErrPrivKey } @@ -166,13 +171,56 @@ func readPrivateKeyECDSA(m map[string]string) (*ecdsa.PrivateKey, error) { return p, nil } +func readPrivateKeyED25519(m map[string]string) (ed25519.PrivateKey, error) { + var p ed25519.PrivateKey + // TODO: validate that the required flags are present + for k, v := range m { + switch k { + case "privatekey": + p1, err := fromBase64([]byte(v)) + if err != nil { + return nil, err + } + if len(p1) != 32 { + return nil, ErrPrivKey + } + // RFC 8080 and Golang's x/crypto/ed25519 differ as to how the + // private keys are represented. RFC 8080 specifies that private + // keys be stored solely as the seed value (p1 above) while the + // ed25519 package represents them as the seed value concatenated + // to the public key, which is derived from the seed value. + // + // ed25519.GenerateKey reads exactly 32 bytes from the passed in + // io.Reader and uses them as the seed. It also derives the + // public key and produces a compatible private key. + _, p, err = ed25519.GenerateKey(bytes.NewReader(p1)) + if err != nil { + return nil, err + } + case "created", "publish", "activate": + /* not used in Go (yet) */ + } + } + return p, nil +} + // parseKey reads a private key from r. It returns a map[string]string, // with the key-value pairs, or an error when the file is not correct. func parseKey(r io.Reader, file string) (map[string]string, error) { - s := scanInit(r) + s, cancel := scanInit(r) m := make(map[string]string) c := make(chan lex) k := "" + defer func() { + cancel() + // zlexer can send up to two tokens, the next one and possibly 1 remainders. + // Do a non-blocking read. + _, ok := <-c + _, ok = <-c + if !ok { + // too bad + } + }() // Start the lexer go klexer(s, c) for l := range c { diff --git a/vendor/github.com/miekg/dns/dnssec_privkey.go b/vendor/github.com/miekg/dns/dnssec_privkey.go index 56f3ea93..46f3215c 100644 --- a/vendor/github.com/miekg/dns/dnssec_privkey.go +++ b/vendor/github.com/miekg/dns/dnssec_privkey.go @@ -7,6 +7,8 @@ import ( "crypto/rsa" "math/big" "strconv" + + "golang.org/x/crypto/ed25519" ) const format = "Private-key-format: v1.3\n" @@ -79,6 +81,12 @@ func (r *DNSKEY) PrivateKeyString(p crypto.PrivateKey) string { "Private_value(x): " + priv + "\n" + "Public_value(y): " + pub + "\n" + case ed25519.PrivateKey: + private := toBase64(p[:32]) + return format + + "Algorithm: " + algorithm + "\n" + + "PrivateKey: " + private + "\n" + default: return "" } diff --git a/vendor/github.com/miekg/dns/doc.go b/vendor/github.com/miekg/dns/doc.go index e38753d7..0389d724 100644 --- a/vendor/github.com/miekg/dns/doc.go +++ b/vendor/github.com/miekg/dns/doc.go @@ -1,7 +1,7 @@ /* Package dns implements a full featured interface to the Domain Name System. Server- and client-side programming is supported. -The package allows complete control over what is send out to the DNS. The package +The package allows complete control over what is sent out to the DNS. The package API follows the less-is-more principle, by presenting a small, clean interface. The package dns supports (asynchronous) querying/replying, incoming/outgoing zone transfers, @@ -14,7 +14,7 @@ Basic usage pattern for creating a new resource record: r := new(dns.MX) r.Hdr = dns.RR_Header{Name: "miek.nl.", Rrtype: dns.TypeMX, - Class: dns.ClassINET, Ttl: 3600} + Class: dns.ClassINET, Ttl: 3600} r.Preference = 10 r.Mx = "mx.miek.nl." @@ -22,16 +22,16 @@ Or directly from a string: mx, err := dns.NewRR("miek.nl. 3600 IN MX 10 mx.miek.nl.") -Or when the default TTL (3600) and class (IN) suit you: +Or when the default origin (.) and TTL (3600) and class (IN) suit you: - mx, err := dns.NewRR("miek.nl. MX 10 mx.miek.nl.") + mx, err := dns.NewRR("miek.nl MX 10 mx.miek.nl") Or even: mx, err := dns.NewRR("$ORIGIN nl.\nmiek 1H IN MX 10 mx.miek") In the DNS messages are exchanged, these messages contain resource -records (sets). Use pattern for creating a message: +records (sets). Use pattern for creating a message: m := new(dns.Msg) m.SetQuestion("miek.nl.", dns.TypeMX) @@ -51,7 +51,7 @@ The following is slightly more verbose, but more flexible: m1.Question = make([]dns.Question, 1) m1.Question[0] = dns.Question{"miek.nl.", dns.TypeMX, dns.ClassINET} -After creating a message it can be send. +After creating a message it can be sent. Basic use pattern for synchronous querying the DNS at a server configured on 127.0.0.1 and port 53: @@ -63,7 +63,23 @@ class) is as easy as setting: c.SingleInflight = true -If these "advanced" features are not needed, a simple UDP query can be send, +More advanced options are available using a net.Dialer and the corresponding API. +For example it is possible to set a timeout, or to specify a source IP address +and port to use for the connection: + + c := new(dns.Client) + laddr := net.UDPAddr{ + IP: net.ParseIP("[::1]"), + Port: 12345, + Zone: "", + } + c.Dialer := &net.Dialer{ + Timeout: 200 * time.Millisecond, + LocalAddr: &laddr, + } + in, rtt, err := c.Exchange(m1, "8.8.8.8:53") + +If these "advanced" features are not needed, a simple UDP query can be sent, with: in, err := dns.Exchange(m1, "127.0.0.1:53") @@ -152,6 +168,11 @@ Basic use pattern when querying with a TSIG name "axfr." (note that these key na must be fully qualified - as they are domain names) and the base64 secret "so6ZGir4GPAqINNh9U5c3A==": +If an incoming message contains a TSIG record it MUST be the last record in +the additional section (RFC2845 3.2). This means that you should make the +call to SetTsig last, right before executing the query. If you make any +changes to the RRset after calling SetTsig() the signature will be incorrect. + c := new(dns.Client) c.TsigSecret = map[string]string{"axfr.": "so6ZGir4GPAqINNh9U5c3A=="} m := new(dns.Msg) diff --git a/vendor/github.com/miekg/dns/edns.go b/vendor/github.com/miekg/dns/edns.go index dbff3714..6f9d2ea3 100644 --- a/vendor/github.com/miekg/dns/edns.go +++ b/vendor/github.com/miekg/dns/edns.go @@ -13,18 +13,18 @@ import ( const ( EDNS0LLQ = 0x1 // long lived queries: http://tools.ietf.org/html/draft-sekar-dns-llq-01 EDNS0UL = 0x2 // update lease draft: http://files.dns-sd.org/draft-sekar-dns-ul.txt - EDNS0NSID = 0x3 // nsid (RFC5001) + EDNS0NSID = 0x3 // nsid (See RFC 5001) EDNS0DAU = 0x5 // DNSSEC Algorithm Understood EDNS0DHU = 0x6 // DS Hash Understood EDNS0N3U = 0x7 // NSEC3 Hash Understood - EDNS0SUBNET = 0x8 // client-subnet (RFC6891) + EDNS0SUBNET = 0x8 // client-subnet (See RFC 7871) EDNS0EXPIRE = 0x9 // EDNS0 expire EDNS0COOKIE = 0xa // EDNS0 Cookie - EDNS0TCPKEEPALIVE = 0xb // EDNS0 tcp keep alive (RFC7828) - EDNS0SUBNETDRAFT = 0x50fa // Don't use! Use EDNS0SUBNET - EDNS0LOCALSTART = 0xFDE9 // Beginning of range reserved for local/experimental use (RFC6891) - EDNS0LOCALEND = 0xFFFE // End of range reserved for local/experimental use (RFC6891) - _DO = 1 << 15 // dnssec ok + EDNS0TCPKEEPALIVE = 0xb // EDNS0 tcp keep alive (See RFC 7828) + EDNS0PADDING = 0xc // EDNS0 padding (See RFC 7830) + EDNS0LOCALSTART = 0xFDE9 // Beginning of range reserved for local/experimental use (See RFC 6891) + EDNS0LOCALEND = 0xFFFE // End of range reserved for local/experimental use (See RFC 6891) + _DO = 1 << 15 // DNSSEC OK ) // OPT is the EDNS0 RR appended to messages to convey extra (meta) information. @@ -57,9 +57,6 @@ func (rr *OPT) String() string { } case *EDNS0_SUBNET: s += "\n; SUBNET: " + o.String() - if o.(*EDNS0_SUBNET).DraftOption { - s += " (draft)" - } case *EDNS0_COOKIE: s += "\n; COOKIE: " + o.String() case *EDNS0_UL: @@ -74,6 +71,8 @@ func (rr *OPT) String() string { s += "\n; NSEC3 HASH UNDERSTOOD: " + o.String() case *EDNS0_LOCAL: s += "\n; LOCAL OPT: " + o.String() + case *EDNS0_PADDING: + s += "\n; PADDING: " + o.String() } } return s @@ -103,15 +102,12 @@ func (rr *OPT) SetVersion(v uint8) { // ExtendedRcode returns the EDNS extended RCODE field (the upper 8 bits of the TTL). func (rr *OPT) ExtendedRcode() int { - return int((rr.Hdr.Ttl&0xFF000000)>>24) + 15 + return int((rr.Hdr.Ttl & 0xFF000000) >> 24) } // SetExtendedRcode sets the EDNS extended RCODE field. func (rr *OPT) SetExtendedRcode(v uint8) { - if v < RcodeBadVers { // Smaller than 16.. Use the 4 bits you have! - return - } - rr.Hdr.Ttl = rr.Hdr.Ttl&0x00FFFFFF | (uint32(v-15) << 24) + rr.Hdr.Ttl = rr.Hdr.Ttl&0x00FFFFFF | (uint32(v) << 24) } // UDPSize returns the UDP buffer size. @@ -182,12 +178,13 @@ func (e *EDNS0_NSID) pack() ([]byte, error) { return h, nil } -func (e *EDNS0_NSID) Option() uint16 { return EDNS0NSID } +// Option implements the EDNS0 interface. +func (e *EDNS0_NSID) Option() uint16 { return EDNS0NSID } // Option returns the option code. func (e *EDNS0_NSID) unpack(b []byte) error { e.Nsid = hex.EncodeToString(b); return nil } func (e *EDNS0_NSID) String() string { return string(e.Nsid) } // EDNS0_SUBNET is the subnet option that is used to give the remote nameserver -// an idea of where the client lives. It can then give back a different +// an idea of where the client lives. See RFC 7871. It can then give back a different // answer depending on the location or network topology. // Basic use pattern for creating an subnet option: // @@ -203,25 +200,19 @@ func (e *EDNS0_NSID) String() string { return string(e.Nsid) } // // e.Address = net.ParseIP("2001:7b8:32a::2") // for IPV6 // o.Option = append(o.Option, e) // -// Note: the spec (draft-ietf-dnsop-edns-client-subnet-00) has some insane logic -// for which netmask applies to the address. This code will parse all the -// available bits when unpacking (up to optlen). When packing it will apply -// SourceNetmask. If you need more advanced logic, patches welcome and good luck. +// This code will parse all the available bits when unpacking (up to optlen). +// When packing it will apply SourceNetmask. If you need more advanced logic, +// patches welcome and good luck. type EDNS0_SUBNET struct { Code uint16 // Always EDNS0SUBNET Family uint16 // 1 for IP, 2 for IP6 SourceNetmask uint8 SourceScope uint8 Address net.IP - DraftOption bool // Set to true if using the old (0x50fa) option code } -func (e *EDNS0_SUBNET) Option() uint16 { - if e.DraftOption { - return EDNS0SUBNETDRAFT - } - return EDNS0SUBNET -} +// Option implements the EDNS0 interface. +func (e *EDNS0_SUBNET) Option() uint16 { return EDNS0SUBNET } func (e *EDNS0_SUBNET) pack() ([]byte, error) { b := make([]byte, 4) @@ -229,6 +220,12 @@ func (e *EDNS0_SUBNET) pack() ([]byte, error) { b[2] = e.SourceNetmask b[3] = e.SourceScope switch e.Family { + case 0: + // "dig" sets AddressFamily to 0 if SourceNetmask is also 0 + // We might don't need to complain either + if e.SourceNetmask != 0 { + return nil, errors.New("dns: bad address family") + } case 1: if e.SourceNetmask > net.IPv4len*8 { return nil, errors.New("dns: bad netmask") @@ -263,6 +260,13 @@ func (e *EDNS0_SUBNET) unpack(b []byte) error { e.SourceNetmask = b[2] e.SourceScope = b[3] switch e.Family { + case 0: + // "dig" sets AddressFamily to 0 if SourceNetmask is also 0 + // It's okay to accept such a packet + if e.SourceNetmask != 0 { + return errors.New("dns: bad address family") + } + e.Address = net.IPv4(0, 0, 0, 0) case 1: if e.SourceNetmask > net.IPv4len*8 || e.SourceScope > net.IPv4len*8 { return errors.New("dns: bad netmask") @@ -332,6 +336,7 @@ func (e *EDNS0_COOKIE) pack() ([]byte, error) { return h, nil } +// Option implements the EDNS0 interface. func (e *EDNS0_COOKIE) Option() uint16 { return EDNS0COOKIE } func (e *EDNS0_COOKIE) unpack(b []byte) error { e.Cookie = hex.EncodeToString(b); return nil } func (e *EDNS0_COOKIE) String() string { return e.Cookie } @@ -353,6 +358,7 @@ type EDNS0_UL struct { Lease uint32 } +// Option implements the EDNS0 interface. func (e *EDNS0_UL) Option() uint16 { return EDNS0UL } func (e *EDNS0_UL) String() string { return strconv.FormatUint(uint64(e.Lease), 10) } @@ -382,6 +388,7 @@ type EDNS0_LLQ struct { LeaseLife uint32 } +// Option implements the EDNS0 interface. func (e *EDNS0_LLQ) Option() uint16 { return EDNS0LLQ } func (e *EDNS0_LLQ) pack() ([]byte, error) { @@ -413,11 +420,13 @@ func (e *EDNS0_LLQ) String() string { return s } +// EDNS0_DUA implements the EDNS0 "DNSSEC Algorithm Understood" option. See RFC 6975. type EDNS0_DAU struct { Code uint16 // Always EDNS0DAU AlgCode []uint8 } +// Option implements the EDNS0 interface. func (e *EDNS0_DAU) Option() uint16 { return EDNS0DAU } func (e *EDNS0_DAU) pack() ([]byte, error) { return e.AlgCode, nil } func (e *EDNS0_DAU) unpack(b []byte) error { e.AlgCode = b; return nil } @@ -434,11 +443,13 @@ func (e *EDNS0_DAU) String() string { return s } +// EDNS0_DHU implements the EDNS0 "DS Hash Understood" option. See RFC 6975. type EDNS0_DHU struct { Code uint16 // Always EDNS0DHU AlgCode []uint8 } +// Option implements the EDNS0 interface. func (e *EDNS0_DHU) Option() uint16 { return EDNS0DHU } func (e *EDNS0_DHU) pack() ([]byte, error) { return e.AlgCode, nil } func (e *EDNS0_DHU) unpack(b []byte) error { e.AlgCode = b; return nil } @@ -455,11 +466,13 @@ func (e *EDNS0_DHU) String() string { return s } +// EDNS0_N3U implements the EDNS0 "NSEC3 Hash Understood" option. See RFC 6975. type EDNS0_N3U struct { Code uint16 // Always EDNS0N3U AlgCode []uint8 } +// Option implements the EDNS0 interface. func (e *EDNS0_N3U) Option() uint16 { return EDNS0N3U } func (e *EDNS0_N3U) pack() ([]byte, error) { return e.AlgCode, nil } func (e *EDNS0_N3U) unpack(b []byte) error { e.AlgCode = b; return nil } @@ -477,11 +490,13 @@ func (e *EDNS0_N3U) String() string { return s } +// EDNS0_EXPIRE implementes the EDNS0 option as described in RFC 7314. type EDNS0_EXPIRE struct { Code uint16 // Always EDNS0EXPIRE Expire uint32 } +// Option implements the EDNS0 interface. func (e *EDNS0_EXPIRE) Option() uint16 { return EDNS0EXPIRE } func (e *EDNS0_EXPIRE) String() string { return strconv.FormatUint(uint64(e.Expire), 10) } @@ -520,6 +535,7 @@ type EDNS0_LOCAL struct { Data []byte } +// Option implements the EDNS0 interface. func (e *EDNS0_LOCAL) Option() uint16 { return e.Code } func (e *EDNS0_LOCAL) String() string { return strconv.FormatInt(int64(e.Code), 10) + ":0x" + hex.EncodeToString(e.Data) @@ -551,6 +567,7 @@ type EDNS0_TCP_KEEPALIVE struct { Timeout uint16 // an idle timeout value for the TCP connection, specified in units of 100 milliseconds, encoded in network byte order. } +// Option implements the EDNS0 interface. func (e *EDNS0_TCP_KEEPALIVE) Option() uint16 { return EDNS0TCPKEEPALIVE } func (e *EDNS0_TCP_KEEPALIVE) pack() ([]byte, error) { @@ -595,3 +612,16 @@ func (e *EDNS0_TCP_KEEPALIVE) String() (s string) { } return } + +// EDNS0_PADDING option is used to add padding to a request/response. The default +// value of padding SHOULD be 0x0 but other values MAY be used, for instance if +// compression is applied before encryption which may break signatures. +type EDNS0_PADDING struct { + Padding []byte +} + +// Option implements the EDNS0 interface. +func (e *EDNS0_PADDING) Option() uint16 { return EDNS0PADDING } +func (e *EDNS0_PADDING) pack() ([]byte, error) { return e.Padding, nil } +func (e *EDNS0_PADDING) unpack(b []byte) error { e.Padding = b; return nil } +func (e *EDNS0_PADDING) String() string { return fmt.Sprintf("%0X", e.Padding) } diff --git a/vendor/github.com/miekg/dns/fuzz.go b/vendor/github.com/miekg/dns/fuzz.go new file mode 100644 index 00000000..a8a09184 --- /dev/null +++ b/vendor/github.com/miekg/dns/fuzz.go @@ -0,0 +1,23 @@ +// +build fuzz + +package dns + +func Fuzz(data []byte) int { + msg := new(Msg) + + if err := msg.Unpack(data); err != nil { + return 0 + } + if _, err := msg.Pack(); err != nil { + return 0 + } + + return 1 +} + +func FuzzNewRR(data []byte) int { + if _, err := NewRR(string(data)); err != nil { + return 0 + } + return 1 +} diff --git a/vendor/github.com/miekg/dns/labels.go b/vendor/github.com/miekg/dns/labels.go index 9538d9c3..760b89e7 100644 --- a/vendor/github.com/miekg/dns/labels.go +++ b/vendor/github.com/miekg/dns/labels.go @@ -1,7 +1,5 @@ package dns -import "strings" - // Holds a bunch of helper functions for dealing with labels. // SplitDomainName splits a name string into it's labels. @@ -44,7 +42,7 @@ func SplitDomainName(s string) (labels []string) { // CompareDomainName compares the names s1 and s2 and // returns how many labels they have in common starting from the *right*. -// The comparison stops at the first inequality. The names are not downcased +// The comparison stops at the first inequality. The names are downcased // before the comparison. // // www.miek.nl. and miek.nl. have two labels in common: miek and nl @@ -52,24 +50,21 @@ func SplitDomainName(s string) (labels []string) { // // s1 and s2 must be syntactically valid domain names. func CompareDomainName(s1, s2 string) (n int) { - s1, s2 = strings.ToLower(s1), strings.ToLower(s2) - s1 = Fqdn(s1) - s2 = Fqdn(s2) + // the first check: root label + if s1 == "." || s2 == "." { + return 0 + } + l1 := Split(s1) l2 := Split(s2) - // the first check: root label - if l1 == nil || l2 == nil { - return - } - j1 := len(l1) - 1 // end i1 := len(l1) - 2 // start j2 := len(l2) - 1 i2 := len(l2) - 2 // the second check can be done here: last/only label // before we fall through into the for-loop below - if s1[l1[j1]:] == s2[l2[j2]:] { + if equal(s1[l1[j1]:], s2[l2[j2]:]) { n++ } else { return @@ -78,7 +73,7 @@ func CompareDomainName(s1, s2 string) (n int) { if i1 < 0 || i2 < 0 { break } - if s1[l1[i1]:l1[j1]] == s2[l2[i2]:l2[j2]] { + if equal(s1[l1[i1]:l1[j1]], s2[l2[i2]:l2[j2]]) { n++ } else { break @@ -169,3 +164,28 @@ func PrevLabel(s string, n int) (i int, start bool) { } return lab[len(lab)-n], false } + +// equal compares a and b while ignoring case. It returns true when equal otherwise false. +func equal(a, b string) bool { + // might be lifted into API function. + la := len(a) + lb := len(b) + if la != lb { + return false + } + + for i := la - 1; i >= 0; i-- { + ai := a[i] + bi := b[i] + if ai >= 'A' && ai <= 'Z' { + ai |= ('a' - 'A') + } + if bi >= 'A' && bi <= 'Z' { + bi |= ('a' - 'A') + } + if ai != bi { + return false + } + } + return true +} diff --git a/vendor/github.com/miekg/dns/msg.go b/vendor/github.com/miekg/dns/msg.go index a348f7c9..276e6b02 100644 --- a/vendor/github.com/miekg/dns/msg.go +++ b/vendor/github.com/miekg/dns/msg.go @@ -26,11 +26,12 @@ const ( maxDomainNameWireOctets = 255 // See RFC 1035 section 2.3.4 ) +// Errors defined in this package. var ( ErrAlg error = &Error{err: "bad algorithm"} // ErrAlg indicates an error with the (DNSSEC) algorithm. ErrAuth error = &Error{err: "bad authentication"} // ErrAuth indicates an error in the TSIG authentication. - ErrBuf error = &Error{err: "buffer size too small"} // ErrBuf indicates that the buffer used it too small for the message. - ErrConnEmpty error = &Error{err: "conn has no connection"} // ErrConnEmpty indicates a connection is being uses before it is initialized. + ErrBuf error = &Error{err: "buffer size too small"} // ErrBuf indicates that the buffer used is too small for the message. + ErrConnEmpty error = &Error{err: "conn has no connection"} // ErrConnEmpty indicates a connection is being used before it is initialized. ErrExtendedRcode error = &Error{err: "bad extended rcode"} // ErrExtendedRcode ... ErrFqdn error = &Error{err: "domain must be fully qualified"} // ErrFqdn indicates that a domain name does not have a closing dot. ErrId error = &Error{err: "id mismatch"} // ErrId indicates there is a mismatch with the message's ID. @@ -57,7 +58,7 @@ var ( // For instance, to make it return a static value: // // dns.Id = func() uint16 { return 3 } -var Id func() uint16 = id +var Id = id var ( idLock sync.Mutex @@ -360,7 +361,7 @@ Loop: case '"', '\\': s = append(s, '\\', b) // presentation-format \X escapes add an extra byte - maxLen += 1 + maxLen++ default: if b < 32 || b >= 127 { // unprintable, use \DDD var buf [3]byte @@ -594,6 +595,13 @@ func UnpackRR(msg []byte, off int) (rr RR, off1 int, err error) { if err != nil { return nil, len(msg), err } + + return UnpackRRWithHeader(h, msg, off) +} + +// UnpackRRWithHeader unpacks the record type specific payload given an existing +// RR_Header. +func UnpackRRWithHeader(h RR_Header, msg []byte, off int) (rr RR, off1 int, err error) { end := off + int(h.Rdlength) if fn, known := typeToUnpack[h.Rrtype]; !known { @@ -611,8 +619,8 @@ func UnpackRR(msg []byte, off int) (rr RR, off1 int, err error) { // If we cannot unpack the whole array, then it will return nil func unpackRRslice(l int, msg []byte, off int) (dst1 []RR, off1 int, err error) { var r RR - // Optimistically make dst be the length that was sent - dst := make([]RR, 0, l) + // Don't pre-allocate, l may be under attacker control + var dst []RR for i := 0; i < l; i++ { off1 := off r, off, err = UnpackRR(msg, off) @@ -810,13 +818,19 @@ func (dns *Msg) Unpack(msg []byte) (err error) { dns.CheckingDisabled = (dh.Bits & _CD) != 0 dns.Rcode = int(dh.Bits & 0xF) + // If we are at the end of the message we should return *just* the + // header. This can still be useful to the caller. 9.9.9.9 sends these + // when responding with REFUSED for instance. if off == len(msg) { - return ErrTruncated + // reset sections before returning + dns.Question, dns.Answer, dns.Ns, dns.Extra = nil, nil, nil, nil + return nil } - // Optimistically use the count given to us in the header - dns.Question = make([]Question, 0, int(dh.Qdcount)) - + // Qdcount, Ancount, Nscount, Arcount can't be trusted, as they are + // attacker controlled. This means we can't use them to pre-allocate + // slices. + dns.Question = nil for i := 0; i < int(dh.Qdcount); i++ { off1 := off var q Question @@ -913,66 +927,70 @@ func (dns *Msg) Len() int { return compressedLen(dns, dns.Compress) } func compressedLen(dns *Msg, compress bool) int { // We always return one more than needed. l := 12 // Message header is always 12 bytes - compression := map[string]int{} + if compress { + compression := map[string]int{} + for _, r := range dns.Question { + l += r.len() + compressionLenHelper(compression, r.Name) + } + l += compressionLenSlice(l, compression, dns.Answer) + l += compressionLenSlice(l, compression, dns.Ns) + l += compressionLenSlice(l, compression, dns.Extra) - for i := 0; i < len(dns.Question); i++ { - l += dns.Question[i].len() - if compress { - compressionLenHelper(compression, dns.Question[i].Name) + return l + } + + for _, r := range dns.Question { + l += r.len() + } + for _, r := range dns.Answer { + if r != nil { + l += r.len() } } - for i := 0; i < len(dns.Answer); i++ { - if dns.Answer[i] == nil { - continue - } - l += dns.Answer[i].len() - if compress { - k, ok := compressionLenSearch(compression, dns.Answer[i].Header().Name) - if ok { - l += 1 - k - } - compressionLenHelper(compression, dns.Answer[i].Header().Name) - k, ok = compressionLenSearchType(compression, dns.Answer[i]) - if ok { - l += 1 - k - } - compressionLenHelperType(compression, dns.Answer[i]) + for _, r := range dns.Ns { + if r != nil { + l += r.len() } } - for i := 0; i < len(dns.Ns); i++ { - if dns.Ns[i] == nil { - continue - } - l += dns.Ns[i].len() - if compress { - k, ok := compressionLenSearch(compression, dns.Ns[i].Header().Name) - if ok { - l += 1 - k - } - compressionLenHelper(compression, dns.Ns[i].Header().Name) - k, ok = compressionLenSearchType(compression, dns.Ns[i]) - if ok { - l += 1 - k - } - compressionLenHelperType(compression, dns.Ns[i]) + for _, r := range dns.Extra { + if r != nil { + l += r.len() } } - for i := 0; i < len(dns.Extra); i++ { - if dns.Extra[i] == nil { + + return l +} + +func compressionLenSlice(len int, c map[string]int, rs []RR) int { + var l int + for _, r := range rs { + if r == nil { continue } - l += dns.Extra[i].len() - if compress { - k, ok := compressionLenSearch(compression, dns.Extra[i].Header().Name) - if ok { - l += 1 - k - } - compressionLenHelper(compression, dns.Extra[i].Header().Name) - k, ok = compressionLenSearchType(compression, dns.Extra[i]) - if ok { - l += 1 - k - } - compressionLenHelperType(compression, dns.Extra[i]) + // track this length, and the global length in len, while taking compression into account for both. + x := r.len() + l += x + len += x + + k, ok := compressionLenSearch(c, r.Header().Name) + if ok { + l += 1 - k + len += 1 - k + } + + if len < maxCompressionOffset { + compressionLenHelper(c, r.Header().Name) + } + + k, ok = compressionLenSearchType(c, r) + if ok { + l += 1 - k + len += 1 - k + } + + if len < maxCompressionOffset { + compressionLenHelperType(c, r) } } return l diff --git a/vendor/github.com/miekg/dns/msg_generate.go b/vendor/github.com/miekg/dns/msg_generate.go index 4d9f81d4..8ba609f7 100644 --- a/vendor/github.com/miekg/dns/msg_generate.go +++ b/vendor/github.com/miekg/dns/msg_generate.go @@ -18,8 +18,7 @@ import ( ) var packageHdr = ` -// *** DO NOT MODIFY *** -// AUTOGENERATED BY go generate from msg_generate.go +// Code generated by "go run msg_generate.go"; DO NOT EDIT. package dns diff --git a/vendor/github.com/miekg/dns/msg_helpers.go b/vendor/github.com/miekg/dns/msg_helpers.go index 615274ab..946d5acb 100644 --- a/vendor/github.com/miekg/dns/msg_helpers.go +++ b/vendor/github.com/miekg/dns/msg_helpers.go @@ -96,7 +96,7 @@ func unpackHeader(msg []byte, off int) (rr RR_Header, off1 int, truncmsg []byte, return hdr, len(msg), msg, err } msg, err = truncateMsgFromRdlength(msg, off, hdr.Rdlength) - return hdr, off, msg, nil + return hdr, off, msg, err } // pack packs an RR header, returning the offset to the end of the header. @@ -406,16 +406,13 @@ Option: } edns = append(edns, e) off += int(optlen) - case EDNS0SUBNET, EDNS0SUBNETDRAFT: + case EDNS0SUBNET: e := new(EDNS0_SUBNET) if err := e.unpack(msg[off : off+int(optlen)]); err != nil { return nil, len(msg), err } edns = append(edns, e) off += int(optlen) - if code == EDNS0SUBNETDRAFT { - e.DraftOption = true - } case EDNS0COOKIE: e := new(EDNS0_COOKIE) if err := e.unpack(msg[off : off+int(optlen)]); err != nil { @@ -458,6 +455,13 @@ Option: } edns = append(edns, e) off += int(optlen) + case EDNS0PADDING: + e := new(EDNS0_PADDING) + if err := e.unpack(msg[off : off+int(optlen)]); err != nil { + return nil, len(msg), err + } + edns = append(edns, e) + off += int(optlen) default: e := new(EDNS0_LOCAL) e.Code = code diff --git a/vendor/github.com/miekg/dns/sanitize.go b/vendor/github.com/miekg/dns/sanitize.go index b489f3f0..c415bdd6 100644 --- a/vendor/github.com/miekg/dns/sanitize.go +++ b/vendor/github.com/miekg/dns/sanitize.go @@ -3,7 +3,7 @@ package dns // Dedup removes identical RRs from rrs. It preserves the original ordering. // The lowest TTL of any duplicates is used in the remaining one. Dedup modifies // rrs. -// m is used to store the RRs temporay. If it is nil a new map will be allocated. +// m is used to store the RRs temporary. If it is nil a new map will be allocated. func Dedup(rrs []RR, m map[string]RR) []RR { if m == nil { m = make(map[string]RR) diff --git a/vendor/github.com/miekg/dns/scan.go b/vendor/github.com/miekg/dns/scan.go index de0db32f..f9cd4740 100644 --- a/vendor/github.com/miekg/dns/scan.go +++ b/vendor/github.com/miekg/dns/scan.go @@ -1,23 +1,14 @@ package dns import ( + "fmt" "io" - "log" "os" + "path/filepath" "strconv" "strings" ) -type debugging bool - -const debug debugging = false - -func (d debugging) Printf(format string, args ...interface{}) { - if d { - log.Printf(format, args...) - } -} - const maxTok = 2048 // Largest token we can return. const maxUint16 = 1<<16 - 1 @@ -38,7 +29,7 @@ const ( zOwner zClass zDirOrigin // $ORIGIN - zDirTtl // $TTL + zDirTTL // $TTL zDirInclude // $INCLUDE zDirGenerate // $GENERATE @@ -51,13 +42,13 @@ const ( zExpectAny // Expect rrtype, ttl or class zExpectAnyNoClass // Expect rrtype or ttl zExpectAnyNoClassBl // The whitespace after _EXPECT_ANY_NOCLASS - zExpectAnyNoTtl // Expect rrtype or class - zExpectAnyNoTtlBl // Whitespace after _EXPECT_ANY_NOTTL + zExpectAnyNoTTL // Expect rrtype or class + zExpectAnyNoTTLBl // Whitespace after _EXPECT_ANY_NOTTL zExpectRrtype // Expect rrtype zExpectRrtypeBl // Whitespace BEFORE rrtype zExpectRdata // The first element of the rdata - zExpectDirTtlBl // Space after directive $TTL - zExpectDirTtl // Directive $TTL + zExpectDirTTLBl // Space after directive $TTL + zExpectDirTTL // Directive $TTL zExpectDirOriginBl // Space after directive $ORIGIN zExpectDirOrigin // Directive $ORIGIN zExpectDirIncludeBl // Space after directive $INCLUDE @@ -105,6 +96,12 @@ type Token struct { Comment string } +// ttlState describes the state necessary to fill in an omitted RR TTL +type ttlState struct { + ttl uint32 // ttl is the current default TTL + isByDirective bool // isByDirective indicates whether ttl was set by a $TTL directive +} + // NewRR reads the RR contained in the string s. Only the first RR is // returned. If s contains no RR, return nil with no error. The class // defaults to IN and TTL defaults to 3600. The full zone file syntax @@ -120,7 +117,8 @@ func NewRR(s string) (RR, error) { // ReadRR reads the RR contained in q. // See NewRR for more documentation. func ReadRR(q io.Reader, filename string) (RR, error) { - r := <-parseZoneHelper(q, ".", filename, 1) + defttl := &ttlState{defaultTtl, false} + r := <-parseZoneHelper(q, ".", filename, defttl, 1) if r == nil { return nil, nil } @@ -132,10 +130,10 @@ func ReadRR(q io.Reader, filename string) (RR, error) { } // ParseZone reads a RFC 1035 style zonefile from r. It returns *Tokens on the -// returned channel, which consist out the parsed RR, a potential comment or an error. -// If there is an error the RR is nil. The string file is only used +// returned channel, each consisting of either a parsed RR and optional comment +// or a nil RR and an error. The string file is only used // in error reporting. The string origin is used as the initial origin, as -// if the file would start with: $ORIGIN origin . +// if the file would start with an $ORIGIN directive. // The directives $INCLUDE, $ORIGIN, $TTL and $GENERATE are supported. // The channel t is closed by ParseZone when the end of r is reached. // @@ -157,25 +155,37 @@ func ReadRR(q io.Reader, filename string) (RR, error) { // The text "; this is comment" is returned in Token.Comment. Comments inside the // RR are discarded. Comments on a line by themselves are discarded too. func ParseZone(r io.Reader, origin, file string) chan *Token { - return parseZoneHelper(r, origin, file, 10000) + return parseZoneHelper(r, origin, file, nil, 10000) } -func parseZoneHelper(r io.Reader, origin, file string, chansize int) chan *Token { +func parseZoneHelper(r io.Reader, origin, file string, defttl *ttlState, chansize int) chan *Token { t := make(chan *Token, chansize) - go parseZone(r, origin, file, t, 0) + go parseZone(r, origin, file, defttl, t, 0) return t } -func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { +func parseZone(r io.Reader, origin, f string, defttl *ttlState, t chan *Token, include int) { defer func() { if include == 0 { close(t) } }() - s := scanInit(r) + s, cancel := scanInit(r) c := make(chan lex) // Start the lexer go zlexer(s, c) + + defer func() { + cancel() + // zlexer can send up to three tokens, the next one and possibly 2 remainders. + // Do a non-blocking read. + _, ok := <-c + _, ok = <-c + _, ok = <-c + if !ok { + // too bad + } + }() // 6 possible beginnings of a line, _ is a space // 0. zRRTYPE -> all omitted until the rrtype // 1. zOwner _ zRrtype -> class/ttl omitted @@ -186,18 +196,16 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { // After detecting these, we know the zRrtype so we can jump to functions // handling the rdata for each of these types. - if origin == "" { - origin = "." - } - origin = Fqdn(origin) - if _, ok := IsDomainName(origin); !ok { - t <- &Token{Error: &ParseError{f, "bad initial origin name", lex{}}} - return + if origin != "" { + origin = Fqdn(origin) + if _, ok := IsDomainName(origin); !ok { + t <- &Token{Error: &ParseError{f, "bad initial origin name", lex{}}} + return + } } st := zExpectOwnerDir // initial state var h RR_Header - var defttl uint32 = defaultTtl var prevName string for l := range c { // Lexer spotted an error already @@ -209,31 +217,25 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { switch st { case zExpectOwnerDir: // We can also expect a directive, like $TTL or $ORIGIN - h.Ttl = defttl + if defttl != nil { + h.Ttl = defttl.ttl + } h.Class = ClassINET switch l.value { case zNewline: st = zExpectOwnerDir case zOwner: h.Name = l.token - if l.token[0] == '@' { - h.Name = origin - prevName = h.Name - st = zExpectOwnerBl - break - } - if h.Name[l.length-1] != '.' { - h.Name = appendOrigin(h.Name, origin) - } - _, ok := IsDomainName(l.token) + name, ok := toAbsoluteName(l.token, origin) if !ok { t <- &Token{Error: &ParseError{f, "bad owner name", l}} return } + h.Name = name prevName = h.Name st = zExpectOwnerBl - case zDirTtl: - st = zExpectDirTtlBl + case zDirTTL: + st = zExpectDirTTLBl case zDirOrigin: st = zExpectDirOriginBl case zDirInclude: @@ -252,15 +254,16 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { // Discard, can happen when there is nothing on the // line except the RR type case zString: - ttl, ok := stringToTtl(l.token) + ttl, ok := stringToTTL(l.token) if !ok { t <- &Token{Error: &ParseError{f, "not a TTL", l}} return } h.Ttl = ttl - // Don't about the defttl, we should take the $TTL value - // defttl = ttl - st = zExpectAnyNoTtlBl + if defttl == nil || !defttl.isByDirective { + defttl = &ttlState{ttl, false} + } + st = zExpectAnyNoTTLBl default: t <- &Token{Error: &ParseError{f, "syntax error at beginning", l}} @@ -278,25 +281,16 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { return } neworigin := origin // There may be optionally a new origin set after the filename, if not use current one - l := <-c - switch l.value { + switch l := <-c; l.value { case zBlank: l := <-c if l.value == zString { - if _, ok := IsDomainName(l.token); !ok || l.length == 0 || l.err { + name, ok := toAbsoluteName(l.token, origin) + if !ok { t <- &Token{Error: &ParseError{f, "bad origin name", l}} return } - // a new origin is specified. - if l.token[l.length-1] != '.' { - if origin != "." { // Prevent .. endings - neworigin = l.token + "." + origin - } else { - neworigin = l.token + origin - } - } else { - neworigin = l.token - } + neworigin = name } case zNewline, zEOF: // Ok @@ -305,24 +299,32 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { return } // Start with the new file - r1, e1 := os.Open(l.token) + includePath := l.token + if !filepath.IsAbs(includePath) { + includePath = filepath.Join(filepath.Dir(f), includePath) + } + r1, e1 := os.Open(includePath) if e1 != nil { - t <- &Token{Error: &ParseError{f, "failed to open `" + l.token + "'", l}} + msg := fmt.Sprintf("failed to open `%s'", l.token) + if !filepath.IsAbs(l.token) { + msg += fmt.Sprintf(" as `%s'", includePath) + } + t <- &Token{Error: &ParseError{f, msg, l}} return } if include+1 > 7 { t <- &Token{Error: &ParseError{f, "too deeply nested $INCLUDE", l}} return } - parseZone(r1, l.token, neworigin, t, include+1) + parseZone(r1, neworigin, includePath, defttl, t, include+1) st = zExpectOwnerDir - case zExpectDirTtlBl: + case zExpectDirTTLBl: if l.value != zBlank { t <- &Token{Error: &ParseError{f, "no blank after $TTL-directive", l}} return } - st = zExpectDirTtl - case zExpectDirTtl: + st = zExpectDirTTL + case zExpectDirTTL: if l.value != zString { t <- &Token{Error: &ParseError{f, "expecting $TTL value, not this...", l}} return @@ -331,12 +333,12 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { t <- &Token{Error: e} return } - ttl, ok := stringToTtl(l.token) + ttl, ok := stringToTTL(l.token) if !ok { t <- &Token{Error: &ParseError{f, "expecting $TTL value, not this...", l}} return } - defttl = ttl + defttl = &ttlState{ttl, true} st = zExpectOwnerDir case zExpectDirOriginBl: if l.value != zBlank { @@ -352,19 +354,12 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { if e, _ := slurpRemainder(c, f); e != nil { t <- &Token{Error: e} } - if _, ok := IsDomainName(l.token); !ok { + name, ok := toAbsoluteName(l.token, origin) + if !ok { t <- &Token{Error: &ParseError{f, "bad origin name", l}} return } - if l.token[l.length-1] != '.' { - if origin != "." { // Prevent .. endings - origin = l.token + "." + origin - } else { - origin = l.token + origin - } - } else { - origin = l.token - } + origin = name st = zExpectOwnerDir case zExpectDirGenerateBl: if l.value != zBlank { @@ -391,20 +386,26 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { case zExpectAny: switch l.value { case zRrtpe: + if defttl == nil { + t <- &Token{Error: &ParseError{f, "missing TTL with no previous value", l}} + return + } h.Rrtype = l.torc st = zExpectRdata case zClass: h.Class = l.torc st = zExpectAnyNoClassBl case zString: - ttl, ok := stringToTtl(l.token) + ttl, ok := stringToTTL(l.token) if !ok { t <- &Token{Error: &ParseError{f, "not a TTL", l}} return } h.Ttl = ttl - // defttl = ttl // don't set the defttl here - st = zExpectAnyNoTtlBl + if defttl == nil || !defttl.isByDirective { + defttl = &ttlState{ttl, false} + } + st = zExpectAnyNoTTLBl default: t <- &Token{Error: &ParseError{f, "expecting RR type, TTL or class, not this...", l}} return @@ -415,13 +416,13 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { return } st = zExpectAnyNoClass - case zExpectAnyNoTtlBl: + case zExpectAnyNoTTLBl: if l.value != zBlank { t <- &Token{Error: &ParseError{f, "no blank before TTL", l}} return } - st = zExpectAnyNoTtl - case zExpectAnyNoTtl: + st = zExpectAnyNoTTL + case zExpectAnyNoTTL: switch l.value { case zClass: h.Class = l.torc @@ -436,13 +437,15 @@ func parseZone(r io.Reader, origin, f string, t chan *Token, include int) { case zExpectAnyNoClass: switch l.value { case zString: - ttl, ok := stringToTtl(l.token) + ttl, ok := stringToTTL(l.token) if !ok { t <- &Token{Error: &ParseError{f, "not a TTL", l}} return } h.Ttl = ttl - // defttl = ttl // don't set the def ttl anymore + if defttl == nil || !defttl.isByDirective { + defttl = &ttlState{ttl, false} + } st = zExpectRrtypeBl case zRrtpe: h.Rrtype = l.torc @@ -505,14 +508,12 @@ func zlexer(s *scan, c chan lex) { if stri >= maxTok { l.token = "token length insufficient for parsing" l.err = true - debug.Printf("[%+v]", l.token) c <- l return } if comi >= maxTok { l.token = "comment length insufficient for parsing" l.err = true - debug.Printf("[%+v]", l.token) c <- l return } @@ -547,7 +548,7 @@ func zlexer(s *scan, c chan lex) { // escape $... start with a \ not a $, so this will work switch l.tokenUpper { case "$TTL": - l.value = zDirTtl + l.value = zDirTTL case "$ORIGIN": l.value = zDirOrigin case "$INCLUDE": @@ -555,7 +556,6 @@ func zlexer(s *scan, c chan lex) { case "$GENERATE": l.value = zDirGenerate } - debug.Printf("[7 %+v]", l.token) c <- l } else { l.value = zString @@ -577,6 +577,7 @@ func zlexer(s *scan, c chan lex) { return } l.value = zRrtpe + rrtype = true l.torc = t } } @@ -597,16 +598,14 @@ func zlexer(s *scan, c chan lex) { } } } - debug.Printf("[6 %+v]", l.token) c <- l } stri = 0 - // I reverse space stuff here + if !space && !commt { l.value = zBlank l.token = " " l.length = 1 - debug.Printf("[5 %+v]", l.token) c <- l } owner = false @@ -629,7 +628,6 @@ func zlexer(s *scan, c chan lex) { l.token = string(str[:stri]) l.tokenUpper = strings.ToUpper(l.token) l.length = stri - debug.Printf("[4 %+v]", l.token) c <- l stri = 0 } @@ -667,7 +665,6 @@ func zlexer(s *scan, c chan lex) { l.tokenUpper = l.token l.length = 1 l.comment = string(com[:comi]) - debug.Printf("[3 %+v %+v]", l.token, l.comment) c <- l l.comment = "" comi = 0 @@ -693,14 +690,12 @@ func zlexer(s *scan, c chan lex) { rrtype = true } } - debug.Printf("[2 %+v]", l.token) c <- l } l.value = zNewline l.token = "\n" l.tokenUpper = l.token l.length = 1 - debug.Printf("[1 %+v]", l.token) c <- l stri = 0 commt = false @@ -746,7 +741,6 @@ func zlexer(s *scan, c chan lex) { l.tokenUpper = strings.ToUpper(l.token) l.length = stri - debug.Printf("[%+v]", l.token) c <- l stri = 0 } @@ -782,7 +776,6 @@ func zlexer(s *scan, c chan lex) { l.token = "extra closing brace" l.tokenUpper = l.token l.err = true - debug.Printf("[%+v]", l.token) c <- l return } @@ -808,7 +801,12 @@ func zlexer(s *scan, c chan lex) { l.tokenUpper = strings.ToUpper(l.token) l.length = stri l.value = zString - debug.Printf("[%+v]", l.token) + c <- l + } + if brace != 0 { + l.token = "unbalanced brace" + l.tokenUpper = l.token + l.err = true c <- l } } @@ -839,8 +837,8 @@ func typeToInt(token string) (uint16, bool) { return uint16(typ), true } -// Parse things like 2w, 2m, etc, Return the time in seconds. -func stringToTtl(token string) (uint32, bool) { +// stringToTTL parses things like 2w, 2m, etc, and returns the time in seconds. +func stringToTTL(token string) (uint32, bool) { s := uint32(0) i := uint32(0) for _, c := range token { @@ -913,6 +911,34 @@ func stringToCm(token string) (e, m uint8, ok bool) { return } +func toAbsoluteName(name, origin string) (absolute string, ok bool) { + // check for an explicit origin reference + if name == "@" { + // require a nonempty origin + if origin == "" { + return "", false + } + return origin, true + } + + // require a valid domain name + _, ok = IsDomainName(name) + if !ok || name == "" { + return "", false + } + + // check if name is already absolute + if name[len(name)-1] == '.' { + return name, true + } + + // require a nonempty origin + if origin == "" { + return "", false + } + return appendOrigin(name, origin), true +} + func appendOrigin(name, origin string) string { if origin == "." { return name + origin diff --git a/vendor/github.com/miekg/dns/scan_rr.go b/vendor/github.com/miekg/dns/scan_rr.go index b8b18fd7..f4ccc842 100644 --- a/vendor/github.com/miekg/dns/scan_rr.go +++ b/vendor/github.com/miekg/dns/scan_rr.go @@ -130,9 +130,10 @@ func setA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { // Dynamic updates. + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + rr.A = net.ParseIP(l.token) if rr.A == nil || l.err { return nil, &ParseError{f, "bad A A", l}, "" @@ -145,9 +146,10 @@ func setAAAA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + rr.AAAA = net.ParseIP(l.token) if rr.AAAA == nil || l.err { return nil, &ParseError{f, "bad AAAA AAAA", l}, "" @@ -161,20 +163,15 @@ func setNS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Ns = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Ns = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad NS Ns", l}, "" } - if rr.Ns[l.length-1] != '.' { - rr.Ns = appendOrigin(rr.Ns, o) - } + rr.Ns = name return rr, nil, "" } @@ -187,17 +184,12 @@ func setPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Ptr = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad PTR Ptr", l}, "" } - if rr.Ptr[l.length-1] != '.' { - rr.Ptr = appendOrigin(rr.Ptr, o) - } + rr.Ptr = name return rr, nil, "" } @@ -207,20 +199,15 @@ func setNSAPPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) l := <-c rr.Ptr = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Ptr = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad NSAP-PTR Ptr", l}, "" } - if rr.Ptr[l.length-1] != '.' { - rr.Ptr = appendOrigin(rr.Ptr, o) - } + rr.Ptr = name return rr, nil, "" } @@ -230,34 +217,26 @@ func setRP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Mbox = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Mbox = o - } else { - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { - return nil, &ParseError{f, "bad RP Mbox", l}, "" - } - if rr.Mbox[l.length-1] != '.' { - rr.Mbox = appendOrigin(rr.Mbox, o) - } + + mbox, mboxOk := toAbsoluteName(l.token, o) + if l.err || !mboxOk { + return nil, &ParseError{f, "bad RP Mbox", l}, "" } + rr.Mbox = mbox + <-c // zBlank l = <-c rr.Txt = l.token - if l.token == "@" { - rr.Txt = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + txt, txtOk := toAbsoluteName(l.token, o) + if l.err || !txtOk { return nil, &ParseError{f, "bad RP Txt", l}, "" } - if rr.Txt[l.length-1] != '.' { - rr.Txt = appendOrigin(rr.Txt, o) - } + rr.Txt = txt + return rr, nil, "" } @@ -267,20 +246,15 @@ func setMR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Mr = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Mr = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad MR Mr", l}, "" } - if rr.Mr[l.length-1] != '.' { - rr.Mr = appendOrigin(rr.Mr, o) - } + rr.Mr = name return rr, nil, "" } @@ -290,20 +264,15 @@ func setMB(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Mb = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Mb = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad MB Mb", l}, "" } - if rr.Mb[l.length-1] != '.' { - rr.Mb = appendOrigin(rr.Mb, o) - } + rr.Mb = name return rr, nil, "" } @@ -313,20 +282,15 @@ func setMG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Mg = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Mg = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad MG Mg", l}, "" } - if rr.Mg[l.length-1] != '.' { - rr.Mg = appendOrigin(rr.Mg, o) - } + rr.Mg = name return rr, nil, "" } @@ -362,34 +326,26 @@ func setMINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Rmail = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Rmail = o - } else { - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { - return nil, &ParseError{f, "bad MINFO Rmail", l}, "" - } - if rr.Rmail[l.length-1] != '.' { - rr.Rmail = appendOrigin(rr.Rmail, o) - } + + rmail, rmailOk := toAbsoluteName(l.token, o) + if l.err || !rmailOk { + return nil, &ParseError{f, "bad MINFO Rmail", l}, "" } + rr.Rmail = rmail + <-c // zBlank l = <-c rr.Email = l.token - if l.token == "@" { - rr.Email = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + email, emailOk := toAbsoluteName(l.token, o) + if l.err || !emailOk { return nil, &ParseError{f, "bad MINFO Email", l}, "" } - if rr.Email[l.length-1] != '.' { - rr.Email = appendOrigin(rr.Email, o) - } + rr.Email = email + return rr, nil, "" } @@ -399,20 +355,15 @@ func setMF(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Mf = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Mf = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad MF Mf", l}, "" } - if rr.Mf[l.length-1] != '.' { - rr.Mf = appendOrigin(rr.Mf, o) - } + rr.Mf = name return rr, nil, "" } @@ -422,20 +373,15 @@ func setMD(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Md = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Md = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad MD Md", l}, "" } - if rr.Md[l.length-1] != '.' { - rr.Md = appendOrigin(rr.Md, o) - } + rr.Md = name return rr, nil, "" } @@ -444,57 +390,54 @@ func setMX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad MX Pref", l}, "" } rr.Preference = uint16(i) + <-c // zBlank l = <-c // zString rr.Mx = l.token - if l.token == "@" { - rr.Mx = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad MX Mx", l}, "" } - if rr.Mx[l.length-1] != '.' { - rr.Mx = appendOrigin(rr.Mx, o) - } + rr.Mx = name + return rr, nil, "" } func setRT(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(RT) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil { return nil, &ParseError{f, "bad RT Preference", l}, "" } rr.Preference = uint16(i) + <-c // zBlank l = <-c // zString rr.Host = l.token - if l.token == "@" { - rr.Host = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad RT Host", l}, "" } - if rr.Host[l.length-1] != '.' { - rr.Host = appendOrigin(rr.Host, o) - } + rr.Host = name + return rr, nil, "" } @@ -503,28 +446,25 @@ func setAFSDB(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad AFSDB Subtype", l}, "" } rr.Subtype = uint16(i) + <-c // zBlank l = <-c // zString rr.Hostname = l.token - if l.token == "@" { - rr.Hostname = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad AFSDB Hostname", l}, "" } - if rr.Hostname[l.length-1] != '.' { - rr.Hostname = appendOrigin(rr.Hostname, o) - } + rr.Hostname = name return rr, nil, "" } @@ -533,9 +473,10 @@ func setX25(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + if l.err { return nil, &ParseError{f, "bad X25 PSDNAddress", l}, "" } @@ -548,28 +489,25 @@ func setKX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad KX Pref", l}, "" } rr.Preference = uint16(i) + <-c // zBlank l = <-c // zString rr.Exchanger = l.token - if l.token == "@" { - rr.Exchanger = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad KX Exchanger", l}, "" } - if rr.Exchanger[l.length-1] != '.' { - rr.Exchanger = appendOrigin(rr.Exchanger, o) - } + rr.Exchanger = name return rr, nil, "" } @@ -579,20 +517,15 @@ func setCNAME(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Target = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Target = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad CNAME Target", l}, "" } - if rr.Target[l.length-1] != '.' { - rr.Target = appendOrigin(rr.Target, o) - } + rr.Target = name return rr, nil, "" } @@ -602,20 +535,15 @@ func setDNAME(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Target = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.Target = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { - return nil, &ParseError{f, "bad CNAME Target", l}, "" - } - if rr.Target[l.length-1] != '.' { - rr.Target = appendOrigin(rr.Target, o) + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { + return nil, &ParseError{f, "bad DNAME Target", l}, "" } + rr.Target = name return rr, nil, "" } @@ -625,35 +553,26 @@ func setSOA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.Ns = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - <-c // zBlank - if l.token == "@" { - rr.Ns = o - } else { - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { - return nil, &ParseError{f, "bad SOA Ns", l}, "" - } - if rr.Ns[l.length-1] != '.' { - rr.Ns = appendOrigin(rr.Ns, o) - } - } + ns, nsOk := toAbsoluteName(l.token, o) + if l.err || !nsOk { + return nil, &ParseError{f, "bad SOA Ns", l}, "" + } + rr.Ns = ns + + <-c // zBlank l = <-c rr.Mbox = l.token - if l.token == "@" { - rr.Mbox = o - } else { - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { - return nil, &ParseError{f, "bad SOA Mbox", l}, "" - } - if rr.Mbox[l.length-1] != '.' { - rr.Mbox = appendOrigin(rr.Mbox, o) - } + + mbox, mboxOk := toAbsoluteName(l.token, o) + if l.err || !mboxOk { + return nil, &ParseError{f, "bad SOA Mbox", l}, "" } + rr.Mbox = mbox + <-c // zBlank var ( @@ -667,10 +586,11 @@ func setSOA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { } if j, e := strconv.ParseUint(l.token, 10, 32); e != nil { if i == 0 { - // Serial should be a number + // Serial must be a number return nil, &ParseError{f, "bad SOA zone parameter", l}, "" } - if v, ok = stringToTtl(l.token); !ok { + // We allow other fields to be unitful duration strings + if v, ok = stringToTTL(l.token); !ok { return nil, &ParseError{f, "bad SOA zone parameter", l}, "" } @@ -702,14 +622,16 @@ func setSRV(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad SRV Priority", l}, "" } rr.Priority = uint16(i) + <-c // zBlank l = <-c // zString i, e = strconv.ParseUint(l.token, 10, 16) @@ -717,6 +639,7 @@ func setSRV(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { return nil, &ParseError{f, "bad SRV Weight", l}, "" } rr.Weight = uint16(i) + <-c // zBlank l = <-c // zString i, e = strconv.ParseUint(l.token, 10, 16) @@ -724,20 +647,16 @@ func setSRV(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { return nil, &ParseError{f, "bad SRV Port", l}, "" } rr.Port = uint16(i) + <-c // zBlank l = <-c // zString rr.Target = l.token - if l.token == "@" { - rr.Target = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad SRV Target", l}, "" } - if rr.Target[l.length-1] != '.' { - rr.Target = appendOrigin(rr.Target, o) - } + rr.Target = name return rr, nil, "" } @@ -746,14 +665,16 @@ func setNAPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad NAPTR Order", l}, "" } rr.Order = uint16(i) + <-c // zBlank l = <-c // zString i, e = strconv.ParseUint(l.token, 10, 16) @@ -761,6 +682,7 @@ func setNAPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { return nil, &ParseError{f, "bad NAPTR Preference", l}, "" } rr.Preference = uint16(i) + // Flags <-c // zBlank l = <-c // _QUOTE @@ -817,21 +739,17 @@ func setNAPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { } else { return nil, &ParseError{f, "bad NAPTR Regexp", l}, "" } + // After quote no space?? <-c // zBlank l = <-c // zString rr.Replacement = l.token - if l.token == "@" { - rr.Replacement = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad NAPTR Replacement", l}, "" } - if rr.Replacement[l.length-1] != '.' { - rr.Replacement = appendOrigin(rr.Replacement, o) - } + rr.Replacement = name return rr, nil, "" } @@ -841,34 +759,26 @@ func setTALINK(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.PreviousName = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } - if l.token == "@" { - rr.PreviousName = o - } else { - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { - return nil, &ParseError{f, "bad TALINK PreviousName", l}, "" - } - if rr.PreviousName[l.length-1] != '.' { - rr.PreviousName = appendOrigin(rr.PreviousName, o) - } + + previousName, previousNameOk := toAbsoluteName(l.token, o) + if l.err || !previousNameOk { + return nil, &ParseError{f, "bad TALINK PreviousName", l}, "" } + rr.PreviousName = previousName + <-c // zBlank l = <-c rr.NextName = l.token - if l.token == "@" { - rr.NextName = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + + nextName, nextNameOk := toAbsoluteName(l.token, o) + if l.err || !nextNameOk { return nil, &ParseError{f, "bad TALINK NextName", l}, "" } - if rr.NextName[l.length-1] != '.' { - rr.NextName = appendOrigin(rr.NextName, o) - } + rr.NextName = nextName + return rr, nil, "" } @@ -880,9 +790,10 @@ func setLOC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.VertPre = 162 // 10 rr.Size = 18 // 1 ok := false + // North l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } i, e := strconv.ParseUint(l.token, 10, 32) @@ -1013,14 +924,16 @@ func setHIP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { // HitLength is not represented l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { return nil, &ParseError{f, "bad HIP PublicKeyAlgorithm", l}, "" } rr.PublicKeyAlgorithm = uint8(i) + <-c // zBlank l = <-c // zString if l.length == 0 || l.err { @@ -1043,19 +956,11 @@ func setHIP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { for l.value != zNewline && l.value != zEOF { switch l.value { case zString: - if l.token == "@" { - xs = append(xs, o) - l = <-c - continue - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad HIP RendezvousServers", l}, "" } - if l.token[l.length-1] != '.' { - l.token = appendOrigin(l.token, o) - } - xs = append(xs, l.token) + xs = append(xs, name) case zBlank: // Ok default: @@ -1072,9 +977,10 @@ func setCERT(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + if v, ok := StringToCertType[l.token]; ok { rr.Type = v } else if i, e := strconv.ParseUint(l.token, 10, 16); e != nil { @@ -1118,6 +1024,56 @@ func setOPENPGPKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, strin return rr, nil, c1 } +func setCSYNC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(CSYNC) + rr.Hdr = h + + l := <-c + if l.length == 0 { // dynamic update rr. + return rr, nil, l.comment + } + j, e := strconv.ParseUint(l.token, 10, 32) + if e != nil { + // Serial must be a number + return nil, &ParseError{f, "bad CSYNC serial", l}, "" + } + rr.Serial = uint32(j) + + <-c // zBlank + + l = <-c + j, e = strconv.ParseUint(l.token, 10, 16) + if e != nil { + // Serial must be a number + return nil, &ParseError{f, "bad CSYNC flags", l}, "" + } + rr.Flags = uint16(j) + + rr.TypeBitMap = make([]uint16, 0) + var ( + k uint16 + ok bool + ) + l = <-c + for l.value != zNewline && l.value != zEOF { + switch l.value { + case zBlank: + // Ok + case zString: + if k, ok = StringToType[l.tokenUpper]; !ok { + if k, ok = typeToInt(l.tokenUpper); !ok { + return nil, &ParseError{f, "bad CSYNC TypeBitMap", l}, "" + } + } + rr.TypeBitMap = append(rr.TypeBitMap, k) + default: + return nil, &ParseError{f, "bad CSYNC TypeBitMap", l}, "" + } + l = <-c + } + return rr, nil, l.comment +} + func setSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { r, e, s := setRRSIG(h, c, o, f) if r != nil { @@ -1129,10 +1085,12 @@ func setSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(RRSIG) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + if t, ok := StringToType[l.tokenUpper]; !ok { if strings.HasPrefix(l.tokenUpper, "TYPE") { t, ok = typeToInt(l.tokenUpper) @@ -1146,6 +1104,7 @@ func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { } else { rr.TypeCovered = t } + <-c // zBlank l = <-c i, err := strconv.ParseUint(l.token, 10, 8) @@ -1153,6 +1112,7 @@ func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { return nil, &ParseError{f, "bad RRSIG Algorithm", l}, "" } rr.Algorithm = uint8(i) + <-c // zBlank l = <-c i, err = strconv.ParseUint(l.token, 10, 8) @@ -1160,6 +1120,7 @@ func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { return nil, &ParseError{f, "bad RRSIG Labels", l}, "" } rr.Labels = uint8(i) + <-c // zBlank l = <-c i, err = strconv.ParseUint(l.token, 10, 32) @@ -1167,6 +1128,7 @@ func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { return nil, &ParseError{f, "bad RRSIG OrigTtl", l}, "" } rr.OrigTtl = uint32(i) + <-c // zBlank l = <-c if i, err := StringToTime(l.token); err != nil { @@ -1180,6 +1142,7 @@ func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { } else { rr.Expiration = i } + <-c // zBlank l = <-c if i, err := StringToTime(l.token); err != nil { @@ -1191,6 +1154,7 @@ func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { } else { rr.Inception = i } + <-c // zBlank l = <-c i, err = strconv.ParseUint(l.token, 10, 16) @@ -1198,25 +1162,22 @@ func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { return nil, &ParseError{f, "bad RRSIG KeyTag", l}, "" } rr.KeyTag = uint16(i) + <-c // zBlank l = <-c rr.SignerName = l.token - if l.token == "@" { - rr.SignerName = o - } else { - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { - return nil, &ParseError{f, "bad RRSIG SignerName", l}, "" - } - if rr.SignerName[l.length-1] != '.' { - rr.SignerName = appendOrigin(rr.SignerName, o) - } + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { + return nil, &ParseError{f, "bad RRSIG SignerName", l}, "" } + rr.SignerName = name + s, e, c1 := endingToString(c, "bad RRSIG Signature", f) if e != nil { return nil, e, c1 } rr.Signature = s + return rr, nil, c1 } @@ -1226,20 +1187,15 @@ func setNSEC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { l := <-c rr.NextDomain = l.token - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } - if l.token == "@" { - rr.NextDomain = o - } else { - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { - return nil, &ParseError{f, "bad NSEC NextDomain", l}, "" - } - if rr.NextDomain[l.length-1] != '.' { - rr.NextDomain = appendOrigin(rr.NextDomain, o) - } + + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { + return nil, &ParseError{f, "bad NSEC NextDomain", l}, "" } + rr.NextDomain = name rr.TypeBitMap = make([]uint16, 0) var ( @@ -1271,9 +1227,10 @@ func setNSEC3(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { return nil, &ParseError{f, "bad NSEC3 Hash", l}, "" @@ -1339,9 +1296,10 @@ func setNSEC3PARAM(h RR_Header, c chan lex, o, f string) (RR, *ParseError, strin rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { return nil, &ParseError{f, "bad NSEC3PARAM Hash", l}, "" @@ -1373,9 +1331,10 @@ func setEUI48(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + if l.length != 17 || l.err { return nil, &ParseError{f, "bad EUI48 Address", l}, "" } @@ -1405,9 +1364,10 @@ func setEUI64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + if l.length != 23 || l.err { return nil, &ParseError{f, "bad EUI64 Address", l}, "" } @@ -1437,9 +1397,10 @@ func setSSHFP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { return nil, &ParseError{f, "bad SSHFP Algorithm", l}, "" @@ -1466,9 +1427,10 @@ func setDNSKEYs(h RR_Header, c chan lex, o, f, typ string) (RR, *ParseError, str rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad " + typ + " Flags", l}, "" @@ -1522,9 +1484,10 @@ func setRKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad RKEY Flags", l}, "" @@ -1577,10 +1540,12 @@ func setNIMLOC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setGPOS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(GPOS) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + _, e := strconv.ParseFloat(l.token, 64) if e != nil || l.err { return nil, &ParseError{f, "bad GPOS Longitude", l}, "" @@ -1606,10 +1571,12 @@ func setGPOS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setDSs(h RR_Header, c chan lex, o, f, typ string) (RR, *ParseError, string) { rr := new(DS) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad " + typ + " KeyTag", l}, "" @@ -1665,10 +1632,12 @@ func setCDS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setTA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(TA) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad TA KeyTag", l}, "" @@ -1703,10 +1672,12 @@ func setTA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setTLSA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(TLSA) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { return nil, &ParseError{f, "bad TLSA Usage", l}, "" @@ -1738,10 +1709,12 @@ func setTLSA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setSMIMEA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(SMIMEA) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { return nil, &ParseError{f, "bad SMIMEA Usage", l}, "" @@ -1773,10 +1746,12 @@ func setSMIMEA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setRFC3597(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(RFC3597) rr.Hdr = h + l := <-c if l.token != "\\#" { return nil, &ParseError{f, "bad RFC3597 Rdata", l}, "" } + <-c // zBlank l = <-c rdlength, e := strconv.Atoi(l.token) @@ -1850,7 +1825,7 @@ func setURI(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { // Dynamic updates. + if l.length == 0 { // dynamic update rr. return rr, nil, "" } @@ -1872,7 +1847,7 @@ func setURI(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { if err != nil { return nil, err, "" } - if len(s) > 1 { + if len(s) != 1 { return nil, &ParseError{f, "bad URI Target", l}, "" } rr.Target = s[0] @@ -1897,9 +1872,10 @@ func setNID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad NID Preference", l}, "" @@ -1920,9 +1896,10 @@ func setL32(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad L32 Preference", l}, "" @@ -1942,31 +1919,25 @@ func setLP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad LP Preference", l}, "" } rr.Preference = uint16(i) + <-c // zBlank l = <-c // zString rr.Fqdn = l.token - if l.length == 0 { - return rr, nil, "" - } - if l.token == "@" { - rr.Fqdn = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + name, nameOk := toAbsoluteName(l.token, o) + if l.err || !nameOk { return nil, &ParseError{f, "bad LP Fqdn", l}, "" } - if rr.Fqdn[l.length-1] != '.' { - rr.Fqdn = appendOrigin(rr.Fqdn, o) - } + rr.Fqdn = name + return rr, nil, "" } @@ -1975,9 +1946,10 @@ func setL64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad L64 Preference", l}, "" @@ -1996,10 +1968,12 @@ func setL64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setUID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(UID) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 32) if e != nil || l.err { return nil, &ParseError{f, "bad UID Uid", l}, "" @@ -2011,10 +1985,12 @@ func setUID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setGID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(GID) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 32) if e != nil || l.err { return nil, &ParseError{f, "bad GID Gid", l}, "" @@ -2026,6 +2002,7 @@ func setGID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { func setUINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(UINFO) rr.Hdr = h + s, e, c1 := endingToTxtSlice(c, "bad UINFO Uinfo", f) if e != nil { return nil, e, c1 @@ -2042,55 +2019,46 @@ func setPX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Hdr = h l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, "" } + i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { return nil, &ParseError{f, "bad PX Preference", l}, "" } rr.Preference = uint16(i) + <-c // zBlank l = <-c // zString rr.Map822 = l.token - if l.length == 0 { - return rr, nil, "" - } - if l.token == "@" { - rr.Map822 = o - return rr, nil, "" - } - _, ok := IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + map822, map822Ok := toAbsoluteName(l.token, o) + if l.err || !map822Ok { return nil, &ParseError{f, "bad PX Map822", l}, "" } - if rr.Map822[l.length-1] != '.' { - rr.Map822 = appendOrigin(rr.Map822, o) - } + rr.Map822 = map822 + <-c // zBlank l = <-c // zString rr.Mapx400 = l.token - if l.token == "@" { - rr.Mapx400 = o - return rr, nil, "" - } - _, ok = IsDomainName(l.token) - if !ok || l.length == 0 || l.err { + mapx400, mapx400Ok := toAbsoluteName(l.token, o) + if l.err || !mapx400Ok { return nil, &ParseError{f, "bad PX Mapx400", l}, "" } - if rr.Mapx400[l.length-1] != '.' { - rr.Mapx400 = appendOrigin(rr.Mapx400, o) - } + rr.Mapx400 = mapx400 + return rr, nil, "" } func setCAA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr := new(CAA) rr.Hdr = h + l := <-c - if l.length == 0 { + if l.length == 0 { // dynamic update rr. return rr, nil, l.comment } + i, err := strconv.ParseUint(l.token, 10, 8) if err != nil || l.err { return nil, &ParseError{f, "bad CAA Flag", l}, "" @@ -2109,13 +2077,58 @@ func setCAA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { if e != nil { return nil, e, "" } - if len(s) > 1 { + if len(s) != 1 { return nil, &ParseError{f, "bad CAA Value", l}, "" } rr.Value = s[0] return rr, nil, c1 } +func setTKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { + rr := new(TKEY) + rr.Hdr = h + + l := <-c + + // Algorithm + if l.value != zString { + return nil, &ParseError{f, "bad TKEY algorithm", l}, "" + } + rr.Algorithm = l.token + <-c // zBlank + + // Get the key length and key values + l = <-c + i, err := strconv.ParseUint(l.token, 10, 8) + if err != nil || l.err { + return nil, &ParseError{f, "bad TKEY key length", l}, "" + } + rr.KeySize = uint16(i) + <-c // zBlank + l = <-c + if l.value != zString { + return nil, &ParseError{f, "bad TKEY key", l}, "" + } + rr.Key = l.token + <-c // zBlank + + // Get the otherdata length and string data + l = <-c + i, err = strconv.ParseUint(l.token, 10, 8) + if err != nil || l.err { + return nil, &ParseError{f, "bad TKEY otherdata length", l}, "" + } + rr.OtherLen = uint16(i) + <-c // zBlank + l = <-c + if l.value != zString { + return nil, &ParseError{f, "bad TKEY otherday", l}, "" + } + rr.OtherData = l.token + + return rr, nil, "" +} + var typeToparserFunc = map[uint16]parserFunc{ TypeAAAA: {setAAAA, false}, TypeAFSDB: {setAFSDB, false}, @@ -2125,6 +2138,7 @@ var typeToparserFunc = map[uint16]parserFunc{ TypeCDNSKEY: {setCDNSKEY, true}, TypeCERT: {setCERT, true}, TypeCNAME: {setCNAME, false}, + TypeCSYNC: {setCSYNC, true}, TypeDHCID: {setDHCID, true}, TypeDLV: {setDLV, true}, TypeDNAME: {setDNAME, false}, @@ -2181,4 +2195,5 @@ var typeToparserFunc = map[uint16]parserFunc{ TypeUINFO: {setUINFO, true}, TypeURI: {setURI, true}, TypeX25: {setX25, false}, + TypeTKEY: {setTKEY, true}, } diff --git a/vendor/github.com/miekg/dns/scanner.go b/vendor/github.com/miekg/dns/scanner.go index c29bc2f3..424e5af9 100644 --- a/vendor/github.com/miekg/dns/scanner.go +++ b/vendor/github.com/miekg/dns/scanner.go @@ -4,6 +4,7 @@ package dns import ( "bufio" + "context" "io" "text/scanner" ) @@ -12,13 +13,18 @@ type scan struct { src *bufio.Reader position scanner.Position eof bool // Have we just seen a eof + ctx context.Context } -func scanInit(r io.Reader) *scan { +func scanInit(r io.Reader) (*scan, context.CancelFunc) { s := new(scan) s.src = bufio.NewReader(r) s.position.Line = 1 - return s + + ctx, cancel := context.WithCancel(context.Background()) + s.ctx = ctx + + return s, cancel } // tokenText returns the next byte from the input @@ -27,6 +33,13 @@ func (s *scan) tokenText() (byte, error) { if err != nil { return c, err } + select { + case <-s.ctx.Done(): + return c, context.Canceled + default: + break + } + // delay the newline handling until the next token is delivered, // fixes off-by-one errors when reporting a parse error. if s.eof == true { diff --git a/vendor/github.com/miekg/dns/server.go b/vendor/github.com/miekg/dns/server.go index 0ca6e008..20ef6b10 100644 --- a/vendor/github.com/miekg/dns/server.go +++ b/vendor/github.com/miekg/dns/server.go @@ -51,7 +51,6 @@ type response struct { udp *net.UDPConn // i/o connection if UDP was used tcp net.Conn // i/o connection if TCP was used udpSession *SessionUDP // oob data to get egress interface right - remoteAddr net.Addr // address of the client writer Writer // writer to output the raw DNS bits } @@ -285,7 +284,7 @@ type Server struct { WriteTimeout time.Duration // TCP idle timeout for multiple queries, if nil, defaults to 8 * time.Second (RFC 5966). IdleTimeout func() time.Duration - // Secret(s) for Tsig map[]. + // Secret(s) for Tsig map[]. The zonename must be in canonical form (lowercase, fqdn, see RFC 4034 Section 6.2). TsigSecret map[string]string // Unsafe instructs the server to disregard any sanity checks and directly hand the message to // the handler. It will specifically not check if the query has the QR bit not set. @@ -297,10 +296,7 @@ type Server struct { // DecorateWriter is optional, allows customization of the process that writes raw DNS messages. DecorateWriter DecorateWriter - // Graceful shutdown handling - - inFlight sync.WaitGroup - + // Shutdown handling lock sync.RWMutex started bool } @@ -412,10 +408,8 @@ func (srv *Server) ActivateAndServe() error { return &Error{err: "bad listeners"} } -// Shutdown gracefully shuts down a server. After a call to Shutdown, ListenAndServe and -// ActivateAndServe will return. All in progress queries are completed before the server -// is taken down. If the Shutdown is taking longer than the reading timeout an error -// is returned. +// Shutdown shuts down a server. After a call to Shutdown, ListenAndServe and +// ActivateAndServe will return. func (srv *Server) Shutdown() error { srv.lock.Lock() if !srv.started { @@ -431,19 +425,7 @@ func (srv *Server) Shutdown() error { if srv.Listener != nil { srv.Listener.Close() } - - fin := make(chan bool) - go func() { - srv.inFlight.Wait() - fin <- true - }() - - select { - case <-time.After(srv.getReadTimeout()): - return &Error{err: "server shutdown is pending"} - case <-fin: - return nil - } + return nil } // getReadTimeout is a helper func to use system timeout if server did not intend to change it. @@ -464,26 +446,13 @@ func (srv *Server) serveTCP(l net.Listener) error { srv.NotifyStartedFunc() } - reader := Reader(&defaultReader{srv}) - if srv.DecorateReader != nil { - reader = srv.DecorateReader(reader) - } - handler := srv.Handler if handler == nil { handler = DefaultServeMux } - rtimeout := srv.getReadTimeout() // deadline is not used here for { rw, err := l.Accept() - if err != nil { - if neterr, ok := err.(net.Error); ok && neterr.Temporary() { - continue - } - return err - } - m, err := reader.ReadTCP(rw, rtimeout) srv.lock.RLock() if !srv.started { srv.lock.RUnlock() @@ -491,10 +460,12 @@ func (srv *Server) serveTCP(l net.Listener) error { } srv.lock.RUnlock() if err != nil { - continue + if neterr, ok := err.(net.Error); ok && neterr.Temporary() { + continue + } + return err } - srv.inFlight.Add(1) - go srv.serve(rw.RemoteAddr(), handler, m, nil, nil, rw) + go srv.serveTCPConn(handler, rw) } } @@ -527,85 +498,102 @@ func (srv *Server) serveUDP(l *net.UDPConn) error { } srv.lock.RUnlock() if err != nil { + if netErr, ok := err.(net.Error); ok && netErr.Temporary() { + continue + } + return err + } + if len(m) < headerSize { continue } - srv.inFlight.Add(1) - go srv.serve(s.RemoteAddr(), handler, m, l, s, nil) + go srv.serveUDPPacket(handler, m, l, s) } } -// Serve a new connection. -func (srv *Server) serve(a net.Addr, h Handler, m []byte, u *net.UDPConn, s *SessionUDP, t net.Conn) { - defer srv.inFlight.Done() +// Serve a new TCP connection. +func (srv *Server) serveTCPConn(h Handler, t net.Conn) { + reader := Reader(&defaultReader{srv}) + if srv.DecorateReader != nil { + reader = srv.DecorateReader(reader) + } - w := &response{tsigSecret: srv.TsigSecret, udp: u, tcp: t, remoteAddr: a, udpSession: s} + w := &response{tsigSecret: srv.TsigSecret, tcp: t} if srv.DecorateWriter != nil { w.writer = srv.DecorateWriter(w) } else { w.writer = w } - q := 0 // counter for the amount of TCP queries we get + defer func() { + if !w.hijacked { + w.Close() + } + }() - reader := Reader(&defaultReader{srv}) - if srv.DecorateReader != nil { - reader = srv.DecorateReader(reader) + idleTimeout := tcpIdleTimeout + if srv.IdleTimeout != nil { + idleTimeout = srv.IdleTimeout() } -Redo: + + timeout := srv.getReadTimeout() + + // TODO(miek): make maxTCPQueries configurable? + for q := 0; q < maxTCPQueries; q++ { + m, err := reader.ReadTCP(t, timeout) + if err != nil { + // TODO(tmthrgd): handle error + break + } + srv.serveDNS(h, m, w) + if w.tcp == nil { + break // Close() was called + } + if w.hijacked { + break // client will call Close() themselves + } + // The first read uses the read timeout, the rest use the + // idle timeout. + timeout = idleTimeout + } +} + +// Serve a new UDP request. +func (srv *Server) serveUDPPacket(h Handler, m []byte, u *net.UDPConn, s *SessionUDP) { + w := &response{tsigSecret: srv.TsigSecret, udp: u, udpSession: s} + if srv.DecorateWriter != nil { + w.writer = srv.DecorateWriter(w) + } else { + w.writer = w + } + srv.serveDNS(h, m, w) +} + +func (srv *Server) serveDNS(h Handler, m []byte, w *response) { req := new(Msg) err := req.Unpack(m) if err != nil { // Send a FormatError back x := new(Msg) x.SetRcodeFormatError(req) w.WriteMsg(x) - goto Exit + return } if !srv.Unsafe && req.Response { - goto Exit + return } w.tsigStatus = nil if w.tsigSecret != nil { if t := req.IsTsig(); t != nil { - secret := t.Hdr.Name - if _, ok := w.tsigSecret[secret]; !ok { - w.tsigStatus = ErrKeyAlg + if secret, ok := w.tsigSecret[t.Hdr.Name]; ok { + w.tsigStatus = TsigVerify(m, secret, "", false) + } else { + w.tsigStatus = ErrSecret } - w.tsigStatus = TsigVerify(m, w.tsigSecret[secret], "", false) w.tsigTimersOnly = false w.tsigRequestMAC = req.Extra[len(req.Extra)-1].(*TSIG).MAC } } h.ServeDNS(w, req) // Writes back to the client - -Exit: - if w.tcp == nil { - return - } - // TODO(miek): make this number configurable? - if q > maxTCPQueries { // close socket after this many queries - w.Close() - return - } - - if w.hijacked { - return // client calls Close() - } - if u != nil { // UDP, "close" and return - w.Close() - return - } - idleTimeout := tcpIdleTimeout - if srv.IdleTimeout != nil { - idleTimeout = srv.IdleTimeout() - } - m, err = reader.ReadTCP(w.tcp, idleTimeout) - if err == nil { - q++ - goto Redo - } - w.Close() - return } func (srv *Server) readTCP(conn net.Conn, timeout time.Duration) ([]byte, error) { @@ -647,11 +635,8 @@ func (srv *Server) readUDP(conn *net.UDPConn, timeout time.Duration) ([]byte, *S conn.SetReadDeadline(time.Now().Add(timeout)) m := make([]byte, srv.UDPSize) n, s, err := ReadFromSessionUDP(conn, m) - if err != nil || n == 0 { - if err != nil { - return nil, nil, err - } - return nil, nil, ErrShortRead + if err != nil { + return nil, nil, err } m = m[:n] return m, s, nil @@ -711,7 +696,12 @@ func (w *response) LocalAddr() net.Addr { } // RemoteAddr implements the ResponseWriter.RemoteAddr method. -func (w *response) RemoteAddr() net.Addr { return w.remoteAddr } +func (w *response) RemoteAddr() net.Addr { + if w.tcp != nil { + return w.tcp.RemoteAddr() + } + return w.udpSession.RemoteAddr() +} // TsigStatus implements the ResponseWriter.TsigStatus method. func (w *response) TsigStatus() error { return w.tsigStatus } diff --git a/vendor/github.com/miekg/dns/tsig.go b/vendor/github.com/miekg/dns/tsig.go index 24013096..4837b4ab 100644 --- a/vendor/github.com/miekg/dns/tsig.go +++ b/vendor/github.com/miekg/dns/tsig.go @@ -208,6 +208,9 @@ func tsigBuffer(msgbuf []byte, rr *TSIG, requestMAC string, timersOnly bool) []b rr.Fudge = 300 // Standard (RFC) default. } + // Replace message ID in header with original ID from TSIG + binary.BigEndian.PutUint16(msgbuf[0:2], rr.OrigId) + if requestMAC != "" { m := new(macWireFmt) m.MACSize = uint16(len(requestMAC) / 2) diff --git a/vendor/github.com/miekg/dns/types.go b/vendor/github.com/miekg/dns/types.go index 53da4755..a779ca8a 100644 --- a/vendor/github.com/miekg/dns/types.go +++ b/vendor/github.com/miekg/dns/types.go @@ -78,6 +78,7 @@ const ( TypeCDS uint16 = 59 TypeCDNSKEY uint16 = 60 TypeOPENPGPKEY uint16 = 61 + TypeCSYNC uint16 = 62 TypeSPF uint16 = 99 TypeUINFO uint16 = 100 TypeUID uint16 = 101 @@ -115,27 +116,27 @@ const ( ClassNONE = 254 ClassANY = 255 - // Message Response Codes. - RcodeSuccess = 0 - RcodeFormatError = 1 - RcodeServerFailure = 2 - RcodeNameError = 3 - RcodeNotImplemented = 4 - RcodeRefused = 5 - RcodeYXDomain = 6 - RcodeYXRrset = 7 - RcodeNXRrset = 8 - RcodeNotAuth = 9 - RcodeNotZone = 10 - RcodeBadSig = 16 // TSIG - RcodeBadVers = 16 // EDNS0 - RcodeBadKey = 17 - RcodeBadTime = 18 - RcodeBadMode = 19 // TKEY - RcodeBadName = 20 - RcodeBadAlg = 21 - RcodeBadTrunc = 22 // TSIG - RcodeBadCookie = 23 // DNS Cookies + // Message Response Codes, see https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml + RcodeSuccess = 0 // NoError - No Error [DNS] + RcodeFormatError = 1 // FormErr - Format Error [DNS] + RcodeServerFailure = 2 // ServFail - Server Failure [DNS] + RcodeNameError = 3 // NXDomain - Non-Existent Domain [DNS] + RcodeNotImplemented = 4 // NotImp - Not Implemented [DNS] + RcodeRefused = 5 // Refused - Query Refused [DNS] + RcodeYXDomain = 6 // YXDomain - Name Exists when it should not [DNS Update] + RcodeYXRrset = 7 // YXRRSet - RR Set Exists when it should not [DNS Update] + RcodeNXRrset = 8 // NXRRSet - RR Set that should exist does not [DNS Update] + RcodeNotAuth = 9 // NotAuth - Server Not Authoritative for zone [DNS Update] + RcodeNotZone = 10 // NotZone - Name not contained in zone [DNS Update/TSIG] + RcodeBadSig = 16 // BADSIG - TSIG Signature Failure [TSIG] + RcodeBadVers = 16 // BADVERS - Bad OPT Version [EDNS0] + RcodeBadKey = 17 // BADKEY - Key not recognized [TSIG] + RcodeBadTime = 18 // BADTIME - Signature out of time window [TSIG] + RcodeBadMode = 19 // BADMODE - Bad TKEY Mode [TKEY] + RcodeBadName = 20 // BADNAME - Duplicate key name [TKEY] + RcodeBadAlg = 21 // BADALG - Algorithm not supported [TKEY] + RcodeBadTrunc = 22 // BADTRUNC - Bad Truncation [TSIG] + RcodeBadCookie = 23 // BADCOOKIE - Bad/missing Server Cookie [DNS Cookies] // Message Opcodes. There is no 3. OpcodeQuery = 0 @@ -164,14 +165,15 @@ const ( _Z = 1 << 6 // Z _AD = 1 << 5 // authticated data _CD = 1 << 4 // checking disabled +) +// Various constants used in the LOC RR, See RFC 1887. +const ( LOC_EQUATOR = 1 << 31 // RFC 1876, Section 2. LOC_PRIMEMERIDIAN = 1 << 31 // RFC 1876, Section 2. - - LOC_HOURS = 60 * 1000 - LOC_DEGREES = 60 * LOC_HOURS - - LOC_ALTITUDEBASE = 100000 + LOC_HOURS = 60 * 1000 + LOC_DEGREES = 60 * LOC_HOURS + LOC_ALTITUDEBASE = 100000 ) // Different Certificate Types, see RFC 4398, Section 2.1 @@ -237,6 +239,7 @@ type ANY struct { func (rr *ANY) String() string { return rr.Hdr.String() } +// CNAME RR. See RFC 1034. type CNAME struct { Hdr RR_Header Target string `dns:"cdomain-name"` @@ -244,6 +247,7 @@ type CNAME struct { func (rr *CNAME) String() string { return rr.Hdr.String() + sprintName(rr.Target) } +// HINFO RR. See RFC 1034. type HINFO struct { Hdr RR_Header Cpu string @@ -254,6 +258,7 @@ func (rr *HINFO) String() string { return rr.Hdr.String() + sprintTxt([]string{rr.Cpu, rr.Os}) } +// MB RR. See RFC 1035. type MB struct { Hdr RR_Header Mb string `dns:"cdomain-name"` @@ -261,6 +266,7 @@ type MB struct { func (rr *MB) String() string { return rr.Hdr.String() + sprintName(rr.Mb) } +// MG RR. See RFC 1035. type MG struct { Hdr RR_Header Mg string `dns:"cdomain-name"` @@ -268,6 +274,7 @@ type MG struct { func (rr *MG) String() string { return rr.Hdr.String() + sprintName(rr.Mg) } +// MINFO RR. See RFC 1035. type MINFO struct { Hdr RR_Header Rmail string `dns:"cdomain-name"` @@ -278,6 +285,7 @@ func (rr *MINFO) String() string { return rr.Hdr.String() + sprintName(rr.Rmail) + " " + sprintName(rr.Email) } +// MR RR. See RFC 1035. type MR struct { Hdr RR_Header Mr string `dns:"cdomain-name"` @@ -287,6 +295,7 @@ func (rr *MR) String() string { return rr.Hdr.String() + sprintName(rr.Mr) } +// MF RR. See RFC 1035. type MF struct { Hdr RR_Header Mf string `dns:"cdomain-name"` @@ -296,6 +305,7 @@ func (rr *MF) String() string { return rr.Hdr.String() + sprintName(rr.Mf) } +// MD RR. See RFC 1035. type MD struct { Hdr RR_Header Md string `dns:"cdomain-name"` @@ -305,6 +315,7 @@ func (rr *MD) String() string { return rr.Hdr.String() + sprintName(rr.Md) } +// MX RR. See RFC 1035. type MX struct { Hdr RR_Header Preference uint16 @@ -315,6 +326,7 @@ func (rr *MX) String() string { return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + " " + sprintName(rr.Mx) } +// AFSDB RR. See RFC 1183. type AFSDB struct { Hdr RR_Header Subtype uint16 @@ -325,6 +337,7 @@ func (rr *AFSDB) String() string { return rr.Hdr.String() + strconv.Itoa(int(rr.Subtype)) + " " + sprintName(rr.Hostname) } +// X25 RR. See RFC 1183, Section 3.1. type X25 struct { Hdr RR_Header PSDNAddress string @@ -334,6 +347,7 @@ func (rr *X25) String() string { return rr.Hdr.String() + rr.PSDNAddress } +// RT RR. See RFC 1183, Section 3.3. type RT struct { Hdr RR_Header Preference uint16 @@ -344,6 +358,7 @@ func (rr *RT) String() string { return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + " " + sprintName(rr.Host) } +// NS RR. See RFC 1035. type NS struct { Hdr RR_Header Ns string `dns:"cdomain-name"` @@ -353,6 +368,7 @@ func (rr *NS) String() string { return rr.Hdr.String() + sprintName(rr.Ns) } +// PTR RR. See RFC 1035. type PTR struct { Hdr RR_Header Ptr string `dns:"cdomain-name"` @@ -362,6 +378,7 @@ func (rr *PTR) String() string { return rr.Hdr.String() + sprintName(rr.Ptr) } +// RP RR. See RFC 1138, Section 2.2. type RP struct { Hdr RR_Header Mbox string `dns:"domain-name"` @@ -372,6 +389,7 @@ func (rr *RP) String() string { return rr.Hdr.String() + rr.Mbox + " " + sprintTxt([]string{rr.Txt}) } +// SOA RR. See RFC 1035. type SOA struct { Hdr RR_Header Ns string `dns:"cdomain-name"` @@ -392,6 +410,7 @@ func (rr *SOA) String() string { " " + strconv.FormatInt(int64(rr.Minttl), 10) } +// TXT RR. See RFC 1035. type TXT struct { Hdr RR_Header Txt []string `dns:"txt"` @@ -524,6 +543,7 @@ func nextByte(b []byte, offset int) (byte, int) { return b[offset+1], 2 } +// SPF RR. See RFC 4408, Section 3.1.1. type SPF struct { Hdr RR_Header Txt []string `dns:"txt"` @@ -531,6 +551,7 @@ type SPF struct { func (rr *SPF) String() string { return rr.Hdr.String() + sprintTxt(rr.Txt) } +// AVC RR. See https://www.iana.org/assignments/dns-parameters/AVC/avc-completed-template. type AVC struct { Hdr RR_Header Txt []string `dns:"txt"` @@ -538,6 +559,7 @@ type AVC struct { func (rr *AVC) String() string { return rr.Hdr.String() + sprintTxt(rr.Txt) } +// SRV RR. See RFC 2782. type SRV struct { Hdr RR_Header Priority uint16 @@ -553,6 +575,7 @@ func (rr *SRV) String() string { strconv.Itoa(int(rr.Port)) + " " + sprintName(rr.Target) } +// NAPTR RR. See RFC 2915. type NAPTR struct { Hdr RR_Header Order uint16 @@ -573,7 +596,7 @@ func (rr *NAPTR) String() string { rr.Replacement } -// The CERT resource record, see RFC 4398. +// CERT RR. See RFC 4398. type CERT struct { Hdr RR_Header Type uint16 @@ -599,7 +622,7 @@ func (rr *CERT) String() string { " " + rr.Certificate } -// The DNAME resource record, see RFC 2672. +// DNAME RR. See RFC 2672. type DNAME struct { Hdr RR_Header Target string `dns:"domain-name"` @@ -609,6 +632,7 @@ func (rr *DNAME) String() string { return rr.Hdr.String() + sprintName(rr.Target) } +// A RR. See RFC 1035. type A struct { Hdr RR_Header A net.IP `dns:"a"` @@ -621,6 +645,7 @@ func (rr *A) String() string { return rr.Hdr.String() + rr.A.String() } +// AAAA RR. See RFC 3596. type AAAA struct { Hdr RR_Header AAAA net.IP `dns:"aaaa"` @@ -633,6 +658,7 @@ func (rr *AAAA) String() string { return rr.Hdr.String() + rr.AAAA.String() } +// PX RR. See RFC 2163. type PX struct { Hdr RR_Header Preference uint16 @@ -644,6 +670,7 @@ func (rr *PX) String() string { return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + " " + sprintName(rr.Map822) + " " + sprintName(rr.Mapx400) } +// GPOS RR. See RFC 1712. type GPOS struct { Hdr RR_Header Longitude string @@ -655,6 +682,7 @@ func (rr *GPOS) String() string { return rr.Hdr.String() + rr.Longitude + " " + rr.Latitude + " " + rr.Altitude } +// LOC RR. See RFC RFC 1876. type LOC struct { Hdr RR_Header Version uint8 @@ -731,11 +759,12 @@ func (rr *LOC) String() string { return s } -// SIG is identical to RRSIG and nowadays only used for SIG(0), RFC2931. +// SIG RR. See RFC 2535. The SIG RR is identical to RRSIG and nowadays only used for SIG(0), See RFC 2931. type SIG struct { RRSIG } +// RRSIG RR. See RFC 4034 and RFC 3755. type RRSIG struct { Hdr RR_Header TypeCovered uint16 @@ -763,6 +792,7 @@ func (rr *RRSIG) String() string { return s } +// NSEC RR. See RFC 4034 and RFC 3755. type NSEC struct { Hdr RR_Header NextDomain string `dns:"domain-name"` @@ -790,14 +820,13 @@ func (rr *NSEC) len() int { return l } -type DLV struct { - DS -} +// DLV RR. See RFC 4431. +type DLV struct{ DS } -type CDS struct { - DS -} +// CDS RR. See RFC 7344. +type CDS struct{ DS } +// DS RR. See RFC 4034 and RFC 3658. type DS struct { Hdr RR_Header KeyTag uint16 @@ -813,6 +842,7 @@ func (rr *DS) String() string { " " + strings.ToUpper(rr.Digest) } +// KX RR. See RFC 2230. type KX struct { Hdr RR_Header Preference uint16 @@ -824,6 +854,7 @@ func (rr *KX) String() string { " " + sprintName(rr.Exchanger) } +// TA RR. See http://www.watson.org/~weiler/INI1999-19.pdf. type TA struct { Hdr RR_Header KeyTag uint16 @@ -839,6 +870,7 @@ func (rr *TA) String() string { " " + strings.ToUpper(rr.Digest) } +// TALINK RR. See https://www.iana.org/assignments/dns-parameters/TALINK/talink-completed-template. type TALINK struct { Hdr RR_Header PreviousName string `dns:"domain-name"` @@ -850,6 +882,7 @@ func (rr *TALINK) String() string { sprintName(rr.PreviousName) + " " + sprintName(rr.NextName) } +// SSHFP RR. See RFC RFC 4255. type SSHFP struct { Hdr RR_Header Algorithm uint8 @@ -863,14 +896,17 @@ func (rr *SSHFP) String() string { " " + strings.ToUpper(rr.FingerPrint) } +// KEY RR. See RFC RFC 2535. type KEY struct { DNSKEY } +// CDNSKEY RR. See RFC 7344. type CDNSKEY struct { DNSKEY } +// DNSKEY RR. See RFC 4034 and RFC 3755. type DNSKEY struct { Hdr RR_Header Flags uint16 @@ -886,6 +922,7 @@ func (rr *DNSKEY) String() string { " " + rr.PublicKey } +// RKEY RR. See https://www.iana.org/assignments/dns-parameters/RKEY/rkey-completed-template. type RKEY struct { Hdr RR_Header Flags uint16 @@ -901,6 +938,7 @@ func (rr *RKEY) String() string { " " + rr.PublicKey } +// NSAPPTR RR. See RFC 1348. type NSAPPTR struct { Hdr RR_Header Ptr string `dns:"domain-name"` @@ -908,6 +946,7 @@ type NSAPPTR struct { func (rr *NSAPPTR) String() string { return rr.Hdr.String() + sprintName(rr.Ptr) } +// NSEC3 RR. See RFC 5155. type NSEC3 struct { Hdr RR_Header Hash uint8 @@ -946,6 +985,7 @@ func (rr *NSEC3) len() int { return l } +// NSEC3PARAM RR. See RFC 5155. type NSEC3PARAM struct { Hdr RR_Header Hash uint8 @@ -964,6 +1004,7 @@ func (rr *NSEC3PARAM) String() string { return s } +// TKEY RR. See RFC 2930. type TKEY struct { Hdr RR_Header Algorithm string `dns:"domain-name"` @@ -972,17 +1013,21 @@ type TKEY struct { Mode uint16 Error uint16 KeySize uint16 - Key string + Key string `dns:"size-hex:KeySize"` OtherLen uint16 - OtherData string + OtherData string `dns:"size-hex:OtherLen"` } +// TKEY has no official presentation format, but this will suffice. func (rr *TKEY) String() string { - // It has no presentation format - return "" + s := "\n;; TKEY PSEUDOSECTION:\n" + s += rr.Hdr.String() + " " + rr.Algorithm + " " + + strconv.Itoa(int(rr.KeySize)) + " " + rr.Key + " " + + strconv.Itoa(int(rr.OtherLen)) + " " + rr.OtherData + return s } -// RFC3597 represents an unknown/generic RR. +// RFC3597 represents an unknown/generic RR. See RFC 3597. type RFC3597 struct { Hdr RR_Header Rdata string `dns:"hex"` @@ -1006,6 +1051,7 @@ func rfc3597Header(h RR_Header) string { return s } +// URI RR. See RFC 7553. type URI struct { Hdr RR_Header Priority uint16 @@ -1018,6 +1064,7 @@ func (rr *URI) String() string { " " + strconv.Itoa(int(rr.Weight)) + " " + sprintTxtOctet(rr.Target) } +// DHCID RR. See RFC 4701. type DHCID struct { Hdr RR_Header Digest string `dns:"base64"` @@ -1025,6 +1072,7 @@ type DHCID struct { func (rr *DHCID) String() string { return rr.Hdr.String() + rr.Digest } +// TLSA RR. See RFC 6698. type TLSA struct { Hdr RR_Header Usage uint8 @@ -1041,6 +1089,7 @@ func (rr *TLSA) String() string { " " + rr.Certificate } +// SMIMEA RR. See RFC 8162. type SMIMEA struct { Hdr RR_Header Usage uint8 @@ -1063,6 +1112,7 @@ func (rr *SMIMEA) String() string { return s } +// HIP RR. See RFC 8005. type HIP struct { Hdr RR_Header HitLength uint8 @@ -1084,6 +1134,7 @@ func (rr *HIP) String() string { return s } +// NINFO RR. See https://www.iana.org/assignments/dns-parameters/NINFO/ninfo-completed-template. type NINFO struct { Hdr RR_Header ZSData []string `dns:"txt"` @@ -1091,6 +1142,7 @@ type NINFO struct { func (rr *NINFO) String() string { return rr.Hdr.String() + sprintTxt(rr.ZSData) } +// NID RR. See RFC RFC 6742. type NID struct { Hdr RR_Header Preference uint16 @@ -1104,6 +1156,7 @@ func (rr *NID) String() string { return s } +// L32 RR, See RFC 6742. type L32 struct { Hdr RR_Header Preference uint16 @@ -1118,6 +1171,7 @@ func (rr *L32) String() string { " " + rr.Locator32.String() } +// L64 RR, See RFC 6742. type L64 struct { Hdr RR_Header Preference uint16 @@ -1131,6 +1185,7 @@ func (rr *L64) String() string { return s } +// LP RR. See RFC 6742. type LP struct { Hdr RR_Header Preference uint16 @@ -1141,6 +1196,7 @@ func (rr *LP) String() string { return rr.Hdr.String() + strconv.Itoa(int(rr.Preference)) + " " + sprintName(rr.Fqdn) } +// EUI48 RR. See RFC 7043. type EUI48 struct { Hdr RR_Header Address uint64 `dns:"uint48"` @@ -1148,6 +1204,7 @@ type EUI48 struct { func (rr *EUI48) String() string { return rr.Hdr.String() + euiToString(rr.Address, 48) } +// EUI64 RR. See RFC 7043. type EUI64 struct { Hdr RR_Header Address uint64 @@ -1155,6 +1212,7 @@ type EUI64 struct { func (rr *EUI64) String() string { return rr.Hdr.String() + euiToString(rr.Address, 64) } +// CAA RR. See RFC 6844. type CAA struct { Hdr RR_Header Flag uint8 @@ -1166,6 +1224,7 @@ func (rr *CAA) String() string { return rr.Hdr.String() + strconv.Itoa(int(rr.Flag)) + " " + rr.Tag + " " + sprintTxtOctet(rr.Value) } +// UID RR. Deprecated, IANA-Reserved. type UID struct { Hdr RR_Header Uid uint32 @@ -1173,6 +1232,7 @@ type UID struct { func (rr *UID) String() string { return rr.Hdr.String() + strconv.FormatInt(int64(rr.Uid), 10) } +// GID RR. Deprecated, IANA-Reserved. type GID struct { Hdr RR_Header Gid uint32 @@ -1180,6 +1240,7 @@ type GID struct { func (rr *GID) String() string { return rr.Hdr.String() + strconv.FormatInt(int64(rr.Gid), 10) } +// UINFO RR. Deprecated, IANA-Reserved. type UINFO struct { Hdr RR_Header Uinfo string @@ -1187,6 +1248,7 @@ type UINFO struct { func (rr *UINFO) String() string { return rr.Hdr.String() + sprintTxt([]string{rr.Uinfo}) } +// EID RR. See http://ana-3.lcs.mit.edu/~jnc/nimrod/dns.txt. type EID struct { Hdr RR_Header Endpoint string `dns:"hex"` @@ -1194,6 +1256,7 @@ type EID struct { func (rr *EID) String() string { return rr.Hdr.String() + strings.ToUpper(rr.Endpoint) } +// NIMLOC RR. See http://ana-3.lcs.mit.edu/~jnc/nimrod/dns.txt. type NIMLOC struct { Hdr RR_Header Locator string `dns:"hex"` @@ -1201,6 +1264,7 @@ type NIMLOC struct { func (rr *NIMLOC) String() string { return rr.Hdr.String() + strings.ToUpper(rr.Locator) } +// OPENPGPKEY RR. See RFC 7929. type OPENPGPKEY struct { Hdr RR_Header PublicKey string `dns:"base64"` @@ -1208,6 +1272,36 @@ type OPENPGPKEY struct { func (rr *OPENPGPKEY) String() string { return rr.Hdr.String() + rr.PublicKey } +// CSYNC RR. See RFC 7477. +type CSYNC struct { + Hdr RR_Header + Serial uint32 + Flags uint16 + TypeBitMap []uint16 `dns:"nsec"` +} + +func (rr *CSYNC) String() string { + s := rr.Hdr.String() + strconv.FormatInt(int64(rr.Serial), 10) + " " + strconv.Itoa(int(rr.Flags)) + + for i := 0; i < len(rr.TypeBitMap); i++ { + s += " " + Type(rr.TypeBitMap[i]).String() + } + return s +} + +func (rr *CSYNC) len() int { + l := rr.Hdr.len() + 4 + 2 + lastwindow := uint32(2 ^ 32 + 1) + for _, t := range rr.TypeBitMap { + window := t / 256 + if uint32(window) != lastwindow { + l += 1 + 32 + } + lastwindow = uint32(window) + } + return l +} + // TimeToString translates the RRSIG's incep. and expir. times to the // string representation used when printing the record. // It takes serial arithmetic (RFC 1982) into account. diff --git a/vendor/github.com/miekg/dns/types_generate.go b/vendor/github.com/miekg/dns/types_generate.go index dd131094..8703cce6 100644 --- a/vendor/github.com/miekg/dns/types_generate.go +++ b/vendor/github.com/miekg/dns/types_generate.go @@ -23,11 +23,11 @@ var skipLen = map[string]struct{}{ "NSEC": {}, "NSEC3": {}, "OPT": {}, + "CSYNC": {}, } var packageHdr = ` -// *** DO NOT MODIFY *** -// AUTOGENERATED BY go generate from type_generate.go +// Code generated by "go run types_generate.go"; DO NOT EDIT. package dns @@ -56,7 +56,6 @@ var TypeToString = map[uint16]string{ `)) var headerFunc = template.Must(template.New("headerFunc").Parse(` -// Header() functions {{range .}} func (rr *{{.}}) Header() *RR_Header { return &rr.Hdr } {{end}} @@ -182,6 +181,8 @@ func main() { fallthrough case st.Tag(i) == `dns:"base64"`: o("l += base64.StdEncoding.DecodedLen(len(rr.%s))\n") + case strings.HasPrefix(st.Tag(i), `dns:"size-hex:`): // this has an extra field where the length is stored + o("l += len(rr.%s)/2\n") case strings.HasPrefix(st.Tag(i), `dns:"size-hex`): fallthrough case st.Tag(i) == `dns:"hex"`: diff --git a/vendor/github.com/miekg/dns/udp.go b/vendor/github.com/miekg/dns/udp.go index af111b9a..a4826ee2 100644 --- a/vendor/github.com/miekg/dns/udp.go +++ b/vendor/github.com/miekg/dns/udp.go @@ -4,8 +4,27 @@ package dns import ( "net" + + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" ) +// This is the required size of the OOB buffer to pass to ReadMsgUDP. +var udpOOBSize = func() int { + // We can't know whether we'll get an IPv4 control message or an + // IPv6 control message ahead of time. To get around this, we size + // the buffer equal to the largest of the two. + + oob4 := ipv4.NewControlMessage(ipv4.FlagDst | ipv4.FlagInterface) + oob6 := ipv6.NewControlMessage(ipv6.FlagDst | ipv6.FlagInterface) + + if len(oob4) > len(oob6) { + return len(oob4) + } + + return len(oob6) +}() + // SessionUDP holds the remote address and the associated // out-of-band data. type SessionUDP struct { @@ -19,7 +38,7 @@ func (s *SessionUDP) RemoteAddr() net.Addr { return s.raddr } // ReadFromSessionUDP acts just like net.UDPConn.ReadFrom(), but returns a session object instead of a // net.UDPAddr. func ReadFromSessionUDP(conn *net.UDPConn, b []byte) (int, *SessionUDP, error) { - oob := make([]byte, 40) + oob := make([]byte, udpOOBSize) n, oobn, _, raddr, err := conn.ReadMsgUDP(b, oob) if err != nil { return n, nil, err @@ -27,8 +46,57 @@ func ReadFromSessionUDP(conn *net.UDPConn, b []byte) (int, *SessionUDP, error) { return n, &SessionUDP{raddr, oob[:oobn]}, err } -// WriteToSessionUDP acts just like net.UDPConn.WritetTo(), but uses a *SessionUDP instead of a net.Addr. +// WriteToSessionUDP acts just like net.UDPConn.WriteTo(), but uses a *SessionUDP instead of a net.Addr. func WriteToSessionUDP(conn *net.UDPConn, b []byte, session *SessionUDP) (int, error) { - n, _, err := conn.WriteMsgUDP(b, session.context, session.raddr) + oob := correctSource(session.context) + n, _, err := conn.WriteMsgUDP(b, oob, session.raddr) return n, err } + +func setUDPSocketOptions(conn *net.UDPConn) error { + // Try setting the flags for both families and ignore the errors unless they + // both error. + err6 := ipv6.NewPacketConn(conn).SetControlMessage(ipv6.FlagDst|ipv6.FlagInterface, true) + err4 := ipv4.NewPacketConn(conn).SetControlMessage(ipv4.FlagDst|ipv4.FlagInterface, true) + if err6 != nil && err4 != nil { + return err4 + } + return nil +} + +// parseDstFromOOB takes oob data and returns the destination IP. +func parseDstFromOOB(oob []byte) net.IP { + // Start with IPv6 and then fallback to IPv4 + // TODO(fastest963): Figure out a way to prefer one or the other. Looking at + // the lvl of the header for a 0 or 41 isn't cross-platform. + cm6 := new(ipv6.ControlMessage) + if cm6.Parse(oob) == nil && cm6.Dst != nil { + return cm6.Dst + } + cm4 := new(ipv4.ControlMessage) + if cm4.Parse(oob) == nil && cm4.Dst != nil { + return cm4.Dst + } + return nil +} + +// correctSource takes oob data and returns new oob data with the Src equal to the Dst +func correctSource(oob []byte) []byte { + dst := parseDstFromOOB(oob) + if dst == nil { + return nil + } + // If the dst is definitely an IPv6, then use ipv6's ControlMessage to + // respond otherwise use ipv4's because ipv6's marshal ignores ipv4 + // addresses. + if dst.To4() == nil { + cm := new(ipv6.ControlMessage) + cm.Src = dst + oob = cm.Marshal() + } else { + cm := new(ipv4.ControlMessage) + cm.Src = dst + oob = cm.Marshal() + } + return oob +} diff --git a/vendor/github.com/miekg/dns/udp_linux.go b/vendor/github.com/miekg/dns/udp_linux.go deleted file mode 100644 index 033df423..00000000 --- a/vendor/github.com/miekg/dns/udp_linux.go +++ /dev/null @@ -1,105 +0,0 @@ -// +build linux,!appengine - -package dns - -// See: -// * http://stackoverflow.com/questions/3062205/setting-the-source-ip-for-a-udp-socket and -// * http://blog.powerdns.com/2012/10/08/on-binding-datagram-udp-sockets-to-the-any-addresses/ -// -// Why do we need this: When listening on 0.0.0.0 with UDP so kernel decides what is the outgoing -// interface, this might not always be the correct one. This code will make sure the egress -// packet's interface matched the ingress' one. - -import ( - "net" - "syscall" -) - -// setUDPSocketOptions sets the UDP socket options. -// This function is implemented on a per platform basis. See udp_*.go for more details -func setUDPSocketOptions(conn *net.UDPConn) error { - sa, err := getUDPSocketName(conn) - if err != nil { - return err - } - switch sa.(type) { - case *syscall.SockaddrInet6: - v6only, err := getUDPSocketOptions6Only(conn) - if err != nil { - return err - } - setUDPSocketOptions6(conn) - if !v6only { - setUDPSocketOptions4(conn) - } - case *syscall.SockaddrInet4: - setUDPSocketOptions4(conn) - } - return nil -} - -// setUDPSocketOptions4 prepares the v4 socket for sessions. -func setUDPSocketOptions4(conn *net.UDPConn) error { - file, err := conn.File() - if err != nil { - return err - } - if err := syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_IP, syscall.IP_PKTINFO, 1); err != nil { - file.Close() - return err - } - // Calling File() above results in the connection becoming blocking, we must fix that. - // See https://github.com/miekg/dns/issues/279 - err = syscall.SetNonblock(int(file.Fd()), true) - if err != nil { - file.Close() - return err - } - file.Close() - return nil -} - -// setUDPSocketOptions6 prepares the v6 socket for sessions. -func setUDPSocketOptions6(conn *net.UDPConn) error { - file, err := conn.File() - if err != nil { - return err - } - if err := syscall.SetsockoptInt(int(file.Fd()), syscall.IPPROTO_IPV6, syscall.IPV6_RECVPKTINFO, 1); err != nil { - file.Close() - return err - } - err = syscall.SetNonblock(int(file.Fd()), true) - if err != nil { - file.Close() - return err - } - file.Close() - return nil -} - -// getUDPSocketOption6Only return true if the socket is v6 only and false when it is v4/v6 combined -// (dualstack). -func getUDPSocketOptions6Only(conn *net.UDPConn) (bool, error) { - file, err := conn.File() - if err != nil { - return false, err - } - // dual stack. See http://stackoverflow.com/questions/1618240/how-to-support-both-ipv4-and-ipv6-connections - v6only, err := syscall.GetsockoptInt(int(file.Fd()), syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY) - if err != nil { - file.Close() - return false, err - } - file.Close() - return v6only == 1, nil -} - -func getUDPSocketName(conn *net.UDPConn) (syscall.Sockaddr, error) { - file, err := conn.File() - if err != nil { - return nil, err - } - defer file.Close() - return syscall.Getsockname(int(file.Fd())) -} diff --git a/vendor/github.com/miekg/dns/udp_other.go b/vendor/github.com/miekg/dns/udp_other.go deleted file mode 100644 index 488a282b..00000000 --- a/vendor/github.com/miekg/dns/udp_other.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build !linux appengine - -package dns - -import ( - "net" -) - -// These do nothing. See udp_linux.go for an example of how to implement this. - -// We tried to adhire to some kind of naming scheme. -func setUDPSocketOptions(conn *net.UDPConn) error { return nil } -func setUDPSocketOptions4(conn *net.UDPConn) error { return nil } -func setUDPSocketOptions6(conn *net.UDPConn) error { return nil } -func getUDPSocketOptions6Only(conn *net.UDPConn) (bool, error) { return false, nil } diff --git a/vendor/github.com/miekg/dns/udp_windows.go b/vendor/github.com/miekg/dns/udp_windows.go index 51e532ac..6778c3c6 100644 --- a/vendor/github.com/miekg/dns/udp_windows.go +++ b/vendor/github.com/miekg/dns/udp_windows.go @@ -4,14 +4,17 @@ package dns import "net" +// SessionUDP holds the remote address type SessionUDP struct { raddr *net.UDPAddr } +// RemoteAddr returns the remote network address. func (s *SessionUDP) RemoteAddr() net.Addr { return s.raddr } // ReadFromSessionUDP acts just like net.UDPConn.ReadFrom(), but returns a session object instead of a // net.UDPAddr. +// TODO(fastest963): Once go1.10 is released, use ReadMsgUDP. func ReadFromSessionUDP(conn *net.UDPConn, b []byte) (int, *SessionUDP, error) { n, raddr, err := conn.ReadFrom(b) if err != nil { @@ -21,9 +24,14 @@ func ReadFromSessionUDP(conn *net.UDPConn, b []byte) (int, *SessionUDP, error) { return n, session, err } -// WriteToSessionUDP acts just like net.UDPConn.WritetTo(), but uses a *SessionUDP instead of a net.Addr. +// WriteToSessionUDP acts just like net.UDPConn.WriteTo(), but uses a *SessionUDP instead of a net.Addr. +// TODO(fastest963): Once go1.10 is released, use WriteMsgUDP. func WriteToSessionUDP(conn *net.UDPConn, b []byte, session *SessionUDP) (int, error) { n, err := conn.WriteTo(b, session.raddr) return n, err } +// TODO(fastest963): Once go1.10 is released and we can use *MsgUDP methods +// use the standard method in udp.go for these. +func setUDPSocketOptions(*net.UDPConn) error { return nil } +func parseDstFromOOB([]byte, net.IP) net.IP { return nil } diff --git a/vendor/github.com/miekg/dns/version.go b/vendor/github.com/miekg/dns/version.go new file mode 100644 index 00000000..e746fee5 --- /dev/null +++ b/vendor/github.com/miekg/dns/version.go @@ -0,0 +1,15 @@ +package dns + +import "fmt" + +// Version is current version of this library. +var Version = V{1, 0, 5} + +// V holds the version of this library. +type V struct { + Major, Minor, Patch int +} + +func (v V) String() string { + return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) +} diff --git a/vendor/github.com/miekg/dns/xfr.go b/vendor/github.com/miekg/dns/xfr.go index 7346deff..5d0ff5c8 100644 --- a/vendor/github.com/miekg/dns/xfr.go +++ b/vendor/github.com/miekg/dns/xfr.go @@ -1,6 +1,7 @@ package dns import ( + "fmt" "time" ) @@ -16,7 +17,7 @@ type Transfer struct { DialTimeout time.Duration // net.DialTimeout, defaults to 2 seconds ReadTimeout time.Duration // net.Conn.SetReadTimeout value for connections, defaults to 2 seconds WriteTimeout time.Duration // net.Conn.SetWriteTimeout value for connections, defaults to 2 seconds - TsigSecret map[string]string // Secret(s) for Tsig map[], zonename must be fully qualified + TsigSecret map[string]string // Secret(s) for Tsig map[], zonename must be in canonical form (lowercase, fqdn, see RFC 4034 Section 6.2) tsigTimersOnly bool } @@ -50,18 +51,18 @@ func (t *Transfer) In(q *Msg, a string) (env chan *Envelope, err error) { env = make(chan *Envelope) go func() { if q.Question[0].Qtype == TypeAXFR { - go t.inAxfr(q.Id, env) + go t.inAxfr(q, env) return } if q.Question[0].Qtype == TypeIXFR { - go t.inIxfr(q.Id, env) + go t.inIxfr(q, env) return } }() return env, nil } -func (t *Transfer) inAxfr(id uint16, c chan *Envelope) { +func (t *Transfer) inAxfr(q *Msg, c chan *Envelope) { first := true defer t.Close() defer close(c) @@ -76,11 +77,15 @@ func (t *Transfer) inAxfr(id uint16, c chan *Envelope) { c <- &Envelope{nil, err} return } - if id != in.Id { + if q.Id != in.Id { c <- &Envelope{in.Answer, ErrId} return } if first { + if in.Rcode != RcodeSuccess { + c <- &Envelope{in.Answer, &Error{err: fmt.Sprintf(errXFR, in.Rcode)}} + return + } if !isSOAFirst(in) { c <- &Envelope{in.Answer, ErrSoa} return @@ -105,9 +110,11 @@ func (t *Transfer) inAxfr(id uint16, c chan *Envelope) { } } -func (t *Transfer) inIxfr(id uint16, c chan *Envelope) { +func (t *Transfer) inIxfr(q *Msg, c chan *Envelope) { serial := uint32(0) // The first serial seen is the current server serial - first := true + axfr := true + n := 0 + qser := q.Ns[0].(*SOA).Serial defer t.Close() defer close(c) timeout := dnsTimeout @@ -121,17 +128,15 @@ func (t *Transfer) inIxfr(id uint16, c chan *Envelope) { c <- &Envelope{nil, err} return } - if id != in.Id { + if q.Id != in.Id { c <- &Envelope{in.Answer, ErrId} return } - if first { - // A single SOA RR signals "no changes" - if len(in.Answer) == 1 && isSOAFirst(in) { - c <- &Envelope{in.Answer, nil} - return - } - + if in.Rcode != RcodeSuccess { + c <- &Envelope{in.Answer, &Error{err: fmt.Sprintf(errXFR, in.Rcode)}} + return + } + if n == 0 { // Check if the returned answer is ok if !isSOAFirst(in) { c <- &Envelope{in.Answer, ErrSoa} @@ -139,21 +144,30 @@ func (t *Transfer) inIxfr(id uint16, c chan *Envelope) { } // This serial is important serial = in.Answer[0].(*SOA).Serial - first = !first + // Check if there are no changes in zone + if qser >= serial { + c <- &Envelope{in.Answer, nil} + return + } } - // Now we need to check each message for SOA records, to see what we need to do - if !first { - t.tsigTimersOnly = true - // If the last record in the IXFR contains the servers' SOA, we should quit - if v, ok := in.Answer[len(in.Answer)-1].(*SOA); ok { + t.tsigTimersOnly = true + for _, rr := range in.Answer { + if v, ok := rr.(*SOA); ok { if v.Serial == serial { - c <- &Envelope{in.Answer, nil} - return + n++ + // quit if it's a full axfr or the the servers' SOA is repeated the third time + if axfr && n == 2 || n == 3 { + c <- &Envelope{in.Answer, nil} + return + } + } else if axfr { + // it's an ixfr + axfr = false } } - c <- &Envelope{in.Answer, nil} } + c <- &Envelope{in.Answer, nil} } } @@ -242,3 +256,5 @@ func isSOALast(in *Msg) bool { } return false } + +const errXFR = "bad xfr rcode: %d" diff --git a/vendor/github.com/miekg/dns/zcompress.go b/vendor/github.com/miekg/dns/zcompress.go index b277978b..c2503204 100644 --- a/vendor/github.com/miekg/dns/zcompress.go +++ b/vendor/github.com/miekg/dns/zcompress.go @@ -1,87 +1,83 @@ -// *** DO NOT MODIFY *** -// AUTOGENERATED BY go generate from compress_generate.go +// Code generated by "go run compress_generate.go"; DO NOT EDIT. package dns func compressionLenHelperType(c map[string]int, r RR) { switch x := r.(type) { - case *PTR: - compressionLenHelper(c, x.Ptr) - case *SOA: - compressionLenHelper(c, x.Ns) - compressionLenHelper(c, x.Mbox) case *AFSDB: compressionLenHelper(c, x.Hostname) + case *CNAME: + compressionLenHelper(c, x.Target) + case *DNAME: + compressionLenHelper(c, x.Target) case *HIP: for i := range x.RendezvousServers { compressionLenHelper(c, x.RendezvousServers[i]) } + case *KX: + compressionLenHelper(c, x.Exchanger) case *LP: compressionLenHelper(c, x.Fqdn) - case *CNAME: - compressionLenHelper(c, x.Target) case *MB: compressionLenHelper(c, x.Mb) + case *MD: + compressionLenHelper(c, x.Md) + case *MF: + compressionLenHelper(c, x.Mf) + case *MG: + compressionLenHelper(c, x.Mg) + case *MINFO: + compressionLenHelper(c, x.Rmail) + compressionLenHelper(c, x.Email) + case *MR: + compressionLenHelper(c, x.Mr) + case *MX: + compressionLenHelper(c, x.Mx) + case *NAPTR: + compressionLenHelper(c, x.Replacement) + case *NS: + compressionLenHelper(c, x.Ns) + case *NSAPPTR: + compressionLenHelper(c, x.Ptr) + case *NSEC: + compressionLenHelper(c, x.NextDomain) + case *PTR: + compressionLenHelper(c, x.Ptr) + case *PX: + compressionLenHelper(c, x.Map822) + compressionLenHelper(c, x.Mapx400) case *RP: compressionLenHelper(c, x.Mbox) compressionLenHelper(c, x.Txt) case *RRSIG: compressionLenHelper(c, x.SignerName) - case *MF: - compressionLenHelper(c, x.Mf) - case *MINFO: - compressionLenHelper(c, x.Rmail) - compressionLenHelper(c, x.Email) + case *RT: + compressionLenHelper(c, x.Host) case *SIG: compressionLenHelper(c, x.SignerName) + case *SOA: + compressionLenHelper(c, x.Ns) + compressionLenHelper(c, x.Mbox) case *SRV: compressionLenHelper(c, x.Target) - case *TSIG: - compressionLenHelper(c, x.Algorithm) - case *KX: - compressionLenHelper(c, x.Exchanger) - case *MG: - compressionLenHelper(c, x.Mg) - case *NSAPPTR: - compressionLenHelper(c, x.Ptr) - case *PX: - compressionLenHelper(c, x.Map822) - compressionLenHelper(c, x.Mapx400) - case *DNAME: - compressionLenHelper(c, x.Target) - case *MR: - compressionLenHelper(c, x.Mr) - case *MX: - compressionLenHelper(c, x.Mx) - case *TKEY: - compressionLenHelper(c, x.Algorithm) - case *NSEC: - compressionLenHelper(c, x.NextDomain) case *TALINK: compressionLenHelper(c, x.PreviousName) compressionLenHelper(c, x.NextName) - case *MD: - compressionLenHelper(c, x.Md) - case *NAPTR: - compressionLenHelper(c, x.Replacement) - case *NS: - compressionLenHelper(c, x.Ns) - case *RT: - compressionLenHelper(c, x.Host) + case *TKEY: + compressionLenHelper(c, x.Algorithm) + case *TSIG: + compressionLenHelper(c, x.Algorithm) } } func compressionLenSearchType(c map[string]int, r RR) (int, bool) { switch x := r.(type) { - case *MG: - k1, ok1 := compressionLenSearch(c, x.Mg) - return k1, ok1 - case *PTR: - k1, ok1 := compressionLenSearch(c, x.Ptr) - return k1, ok1 case *AFSDB: k1, ok1 := compressionLenSearch(c, x.Hostname) return k1, ok1 + case *CNAME: + k1, ok1 := compressionLenSearch(c, x.Target) + return k1, ok1 case *MB: k1, ok1 := compressionLenSearch(c, x.Mb) return k1, ok1 @@ -91,18 +87,8 @@ func compressionLenSearchType(c map[string]int, r RR) (int, bool) { case *MF: k1, ok1 := compressionLenSearch(c, x.Mf) return k1, ok1 - case *NS: - k1, ok1 := compressionLenSearch(c, x.Ns) - return k1, ok1 - case *RT: - k1, ok1 := compressionLenSearch(c, x.Host) - return k1, ok1 - case *SOA: - k1, ok1 := compressionLenSearch(c, x.Ns) - k2, ok2 := compressionLenSearch(c, x.Mbox) - return k1 + k2, ok1 && ok2 - case *CNAME: - k1, ok1 := compressionLenSearch(c, x.Target) + case *MG: + k1, ok1 := compressionLenSearch(c, x.Mg) return k1, ok1 case *MINFO: k1, ok1 := compressionLenSearch(c, x.Rmail) @@ -114,6 +100,19 @@ func compressionLenSearchType(c map[string]int, r RR) (int, bool) { case *MX: k1, ok1 := compressionLenSearch(c, x.Mx) return k1, ok1 + case *NS: + k1, ok1 := compressionLenSearch(c, x.Ns) + return k1, ok1 + case *PTR: + k1, ok1 := compressionLenSearch(c, x.Ptr) + return k1, ok1 + case *RT: + k1, ok1 := compressionLenSearch(c, x.Host) + return k1, ok1 + case *SOA: + k1, ok1 := compressionLenSearch(c, x.Ns) + k2, ok2 := compressionLenSearch(c, x.Mbox) + return k1 + k2, ok1 && ok2 } return 0, false } diff --git a/vendor/github.com/miekg/dns/zmsg.go b/vendor/github.com/miekg/dns/zmsg.go index 418fb1fe..0d1f6f4d 100644 --- a/vendor/github.com/miekg/dns/zmsg.go +++ b/vendor/github.com/miekg/dns/zmsg.go @@ -1,5 +1,4 @@ -// *** DO NOT MODIFY *** -// AUTOGENERATED BY go generate from msg_generate.go +// Code generated by "go run msg_generate.go"; DO NOT EDIT. package dns @@ -189,6 +188,28 @@ func (rr *CNAME) pack(msg []byte, off int, compression map[string]int, compress return off, nil } +func (rr *CSYNC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { + off, err := rr.Hdr.pack(msg, off, compression, compress) + if err != nil { + return off, err + } + headerEnd := off + off, err = packUint32(rr.Serial, msg, off) + if err != nil { + return off, err + } + off, err = packUint16(rr.Flags, msg, off) + if err != nil { + return off, err + } + off, err = packDataNsec(rr.TypeBitMap, msg, off) + if err != nil { + return off, err + } + rr.Header().Rdlength = uint16(off - headerEnd) + return off, nil +} + func (rr *DHCID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { off, err := rr.Hdr.pack(msg, off, compression, compress) if err != nil { @@ -1303,7 +1324,7 @@ func (rr *TKEY) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - off, err = packString(rr.Key, msg, off) + off, err = packStringHex(rr.Key, msg, off) if err != nil { return off, err } @@ -1311,7 +1332,7 @@ func (rr *TKEY) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - off, err = packString(rr.OtherData, msg, off) + off, err = packStringHex(rr.OtherData, msg, off) if err != nil { return off, err } @@ -1721,6 +1742,37 @@ func unpackCNAME(h RR_Header, msg []byte, off int) (RR, int, error) { return rr, off, err } +func unpackCSYNC(h RR_Header, msg []byte, off int) (RR, int, error) { + rr := new(CSYNC) + rr.Hdr = h + if noRdata(h) { + return rr, off, nil + } + var err error + rdStart := off + _ = rdStart + + rr.Serial, off, err = unpackUint32(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.Flags, off, err = unpackUint16(msg, off) + if err != nil { + return rr, off, err + } + if off == len(msg) { + return rr, off, nil + } + rr.TypeBitMap, off, err = unpackDataNsec(msg, off) + if err != nil { + return rr, off, err + } + return rr, off, err +} + func unpackDHCID(h RR_Header, msg []byte, off int) (RR, int, error) { rr := new(DHCID) rr.Hdr = h @@ -3265,13 +3317,10 @@ func unpackTKEY(h RR_Header, msg []byte, off int) (RR, int, error) { if off == len(msg) { return rr, off, nil } - rr.Key, off, err = unpackString(msg, off) + rr.Key, off, err = unpackStringHex(msg, off, off+int(rr.KeySize)) if err != nil { return rr, off, err } - if off == len(msg) { - return rr, off, nil - } rr.OtherLen, off, err = unpackUint16(msg, off) if err != nil { return rr, off, err @@ -3279,7 +3328,7 @@ func unpackTKEY(h RR_Header, msg []byte, off int) (RR, int, error) { if off == len(msg) { return rr, off, nil } - rr.OtherData, off, err = unpackString(msg, off) + rr.OtherData, off, err = unpackStringHex(msg, off, off+int(rr.OtherLen)) if err != nil { return rr, off, err } @@ -3504,6 +3553,7 @@ var typeToUnpack = map[uint16]func(RR_Header, []byte, int) (RR, int, error){ TypeCDS: unpackCDS, TypeCERT: unpackCERT, TypeCNAME: unpackCNAME, + TypeCSYNC: unpackCSYNC, TypeDHCID: unpackDHCID, TypeDLV: unpackDLV, TypeDNAME: unpackDNAME, diff --git a/vendor/github.com/miekg/dns/ztypes.go b/vendor/github.com/miekg/dns/ztypes.go index 3e534f12..abd75dd9 100644 --- a/vendor/github.com/miekg/dns/ztypes.go +++ b/vendor/github.com/miekg/dns/ztypes.go @@ -1,5 +1,4 @@ -// *** DO NOT MODIFY *** -// AUTOGENERATED BY go generate from type_generate.go +// Code generated by "go run types_generate.go"; DO NOT EDIT. package dns @@ -20,6 +19,7 @@ var TypeToRR = map[uint16]func() RR{ TypeCDS: func() RR { return new(CDS) }, TypeCERT: func() RR { return new(CERT) }, TypeCNAME: func() RR { return new(CNAME) }, + TypeCSYNC: func() RR { return new(CSYNC) }, TypeDHCID: func() RR { return new(DHCID) }, TypeDLV: func() RR { return new(DLV) }, TypeDNAME: func() RR { return new(DNAME) }, @@ -94,6 +94,7 @@ var TypeToString = map[uint16]string{ TypeCDS: "CDS", TypeCERT: "CERT", TypeCNAME: "CNAME", + TypeCSYNC: "CSYNC", TypeDHCID: "DHCID", TypeDLV: "DLV", TypeDNAME: "DNAME", @@ -163,7 +164,6 @@ var TypeToString = map[uint16]string{ TypeNSAPPTR: "NSAP-PTR", } -// Header() functions func (rr *A) Header() *RR_Header { return &rr.Hdr } func (rr *AAAA) Header() *RR_Header { return &rr.Hdr } func (rr *AFSDB) Header() *RR_Header { return &rr.Hdr } @@ -174,6 +174,7 @@ func (rr *CDNSKEY) Header() *RR_Header { return &rr.Hdr } func (rr *CDS) Header() *RR_Header { return &rr.Hdr } func (rr *CERT) Header() *RR_Header { return &rr.Hdr } func (rr *CNAME) Header() *RR_Header { return &rr.Hdr } +func (rr *CSYNC) Header() *RR_Header { return &rr.Hdr } func (rr *DHCID) Header() *RR_Header { return &rr.Hdr } func (rr *DLV) Header() *RR_Header { return &rr.Hdr } func (rr *DNAME) Header() *RR_Header { return &rr.Hdr } @@ -346,7 +347,7 @@ func (rr *HIP) len() int { l++ // HitLength l++ // PublicKeyAlgorithm l += 2 // PublicKeyLength - l += len(rr.Hit)/2 + 1 + l += len(rr.Hit) / 2 l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) for _, x := range rr.RendezvousServers { l += len(x) + 1 @@ -469,7 +470,7 @@ func (rr *NSEC3PARAM) len() int { l++ // Flags l += 2 // Iterations l++ // SaltLength - l += len(rr.Salt)/2 + 1 + l += len(rr.Salt) / 2 return l } func (rr *OPENPGPKEY) len() int { @@ -590,9 +591,9 @@ func (rr *TKEY) len() int { l += 2 // Mode l += 2 // Error l += 2 // KeySize - l += len(rr.Key) + 1 + l += len(rr.Key) / 2 l += 2 // OtherLen - l += len(rr.OtherData) + 1 + l += len(rr.OtherData) / 2 return l } func (rr *TLSA) len() int { @@ -609,11 +610,11 @@ func (rr *TSIG) len() int { l += 6 // TimeSigned l += 2 // Fudge l += 2 // MACSize - l += len(rr.MAC)/2 + 1 + l += len(rr.MAC) / 2 l += 2 // OrigId l += 2 // Error l += 2 // OtherLen - l += len(rr.OtherData)/2 + 1 + l += len(rr.OtherData) / 2 return l } func (rr *TXT) len() int { @@ -673,6 +674,11 @@ func (rr *CERT) copy() RR { func (rr *CNAME) copy() RR { return &CNAME{*rr.Hdr.copyHeader(), rr.Target} } +func (rr *CSYNC) copy() RR { + TypeBitMap := make([]uint16, len(rr.TypeBitMap)) + copy(TypeBitMap, rr.TypeBitMap) + return &CSYNC{*rr.Hdr.copyHeader(), rr.Serial, rr.Flags, TypeBitMap} +} func (rr *DHCID) copy() RR { return &DHCID{*rr.Hdr.copyHeader(), rr.Digest} } diff --git a/vendor/github.com/mitchellh/mapstructure/README.md b/vendor/github.com/mitchellh/mapstructure/README.md index 659d6885..0018dc7d 100644 --- a/vendor/github.com/mitchellh/mapstructure/README.md +++ b/vendor/github.com/mitchellh/mapstructure/README.md @@ -1,4 +1,4 @@ -# mapstructure +# mapstructure [![Godoc](https://godoc.org/github.com/mitchellh/mapstructure?status.svg)](https://godoc.org/github.com/mitchellh/mapstructure) mapstructure is a Go library for decoding generic map values to structures and vice versa, while providing helpful error handling. diff --git a/vendor/github.com/mitchellh/mapstructure/decode_hooks.go b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go index 115ae67c..2a727575 100644 --- a/vendor/github.com/mitchellh/mapstructure/decode_hooks.go +++ b/vendor/github.com/mitchellh/mapstructure/decode_hooks.go @@ -38,12 +38,6 @@ func DecodeHookExec( raw DecodeHookFunc, from reflect.Type, to reflect.Type, data interface{}) (interface{}, error) { - // Build our arguments that reflect expects - argVals := make([]reflect.Value, 3) - argVals[0] = reflect.ValueOf(from) - argVals[1] = reflect.ValueOf(to) - argVals[2] = reflect.ValueOf(data) - switch f := typedDecodeHook(raw).(type) { case DecodeHookFuncType: return f(from, to, data) @@ -121,6 +115,30 @@ func StringToTimeDurationHookFunc() DecodeHookFunc { } } +// StringToTimeHookFunc returns a DecodeHookFunc that converts +// strings to time.Time. +func StringToTimeHookFunc(layout string) DecodeHookFunc { + return func( + f reflect.Type, + t reflect.Type, + data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + if t != reflect.TypeOf(time.Time{}) { + return data, nil + } + + // Convert it by parsing + return time.Parse(layout, data.(string)) + } +} + +// WeaklyTypedHook is a DecodeHookFunc which adds support for weak typing to +// the decoder. +// +// Note that this is significantly different from the WeaklyTypedInput option +// of the DecoderConfig. func WeaklyTypedHook( f reflect.Kind, t reflect.Kind, @@ -132,9 +150,8 @@ func WeaklyTypedHook( case reflect.Bool: if dataVal.Bool() { return "1", nil - } else { - return "0", nil } + return "0", nil case reflect.Float32: return strconv.FormatFloat(dataVal.Float(), 'f', -1, 64), nil case reflect.Int: diff --git a/vendor/github.com/mitchellh/mapstructure/mapstructure.go b/vendor/github.com/mitchellh/mapstructure/mapstructure.go index 6dee0ef0..aaf12a29 100644 --- a/vendor/github.com/mitchellh/mapstructure/mapstructure.go +++ b/vendor/github.com/mitchellh/mapstructure/mapstructure.go @@ -1,5 +1,5 @@ -// The mapstructure package exposes functionality to convert an -// arbitrary map[string]interface{} into a native Go structure. +// Package mapstructure exposes functionality to convert an arbitrary +// map[string]interface{} into a native Go structure. // // The Go structure can be arbitrarily complex, containing slices, // other structs, etc. and the decoder will properly decode nested @@ -32,7 +32,12 @@ import ( // both. type DecodeHookFunc interface{} +// DecodeHookFuncType is a DecodeHookFunc which has complete information about +// the source and target types. type DecodeHookFuncType func(reflect.Type, reflect.Type, interface{}) (interface{}, error) + +// DecodeHookFuncKind is a DecodeHookFunc which knows only the Kinds of the +// source and target types. type DecodeHookFuncKind func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error) // DecoderConfig is the configuration that is used to create a new decoder @@ -109,12 +114,12 @@ type Metadata struct { Unused []string } -// Decode takes a map and uses reflection to convert it into the -// given Go native structure. val must be a pointer to a struct. -func Decode(m interface{}, rawVal interface{}) error { +// Decode takes an input structure and uses reflection to translate it to +// the output structure. output must be a pointer to a map or struct. +func Decode(input interface{}, output interface{}) error { config := &DecoderConfig{ Metadata: nil, - Result: rawVal, + Result: output, } decoder, err := NewDecoder(config) @@ -122,7 +127,7 @@ func Decode(m interface{}, rawVal interface{}) error { return err } - return decoder.Decode(m) + return decoder.Decode(input) } // WeakDecode is the same as Decode but is shorthand to enable @@ -142,6 +147,40 @@ func WeakDecode(input, output interface{}) error { return decoder.Decode(input) } +// DecodeMetadata is the same as Decode, but is shorthand to +// enable metadata collection. See DecoderConfig for more info. +func DecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error { + config := &DecoderConfig{ + Metadata: metadata, + Result: output, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + +// WeakDecodeMetadata is the same as Decode, but is shorthand to +// enable both WeaklyTypedInput and metadata collection. See +// DecoderConfig for more info. +func WeakDecodeMetadata(input interface{}, output interface{}, metadata *Metadata) error { + config := &DecoderConfig{ + Metadata: metadata, + Result: output, + WeaklyTypedInput: true, + } + + decoder, err := NewDecoder(config) + if err != nil { + return err + } + + return decoder.Decode(input) +} + // NewDecoder returns a new decoder for the given configuration. Once // a decoder has been returned, the same configuration must not be used // again. @@ -179,68 +218,81 @@ func NewDecoder(config *DecoderConfig) (*Decoder, error) { // Decode decodes the given raw interface to the target pointer specified // by the configuration. -func (d *Decoder) Decode(raw interface{}) error { - return d.decode("", raw, reflect.ValueOf(d.config.Result).Elem()) +func (d *Decoder) Decode(input interface{}) error { + return d.decode("", input, reflect.ValueOf(d.config.Result).Elem()) } // Decodes an unknown data type into a specific reflection value. -func (d *Decoder) decode(name string, data interface{}, val reflect.Value) error { - if data == nil { - // If the data is nil, then we don't set anything. +func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) error { + if input == nil { + // If the data is nil, then we don't set anything, unless ZeroFields is set + // to true. + if d.config.ZeroFields { + outVal.Set(reflect.Zero(outVal.Type())) + + if d.config.Metadata != nil && name != "" { + d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) + } + } return nil } - dataVal := reflect.ValueOf(data) - if !dataVal.IsValid() { - // If the data value is invalid, then we just set the value + inputVal := reflect.ValueOf(input) + if !inputVal.IsValid() { + // If the input value is invalid, then we just set the value // to be the zero value. - val.Set(reflect.Zero(val.Type())) + outVal.Set(reflect.Zero(outVal.Type())) + if d.config.Metadata != nil && name != "" { + d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) + } return nil } if d.config.DecodeHook != nil { - // We have a DecodeHook, so let's pre-process the data. + // We have a DecodeHook, so let's pre-process the input. var err error - data, err = DecodeHookExec( + input, err = DecodeHookExec( d.config.DecodeHook, - dataVal.Type(), val.Type(), data) + inputVal.Type(), outVal.Type(), input) if err != nil { return fmt.Errorf("error decoding '%s': %s", name, err) } } var err error - dataKind := getKind(val) - switch dataKind { + inputKind := getKind(outVal) + switch inputKind { case reflect.Bool: - err = d.decodeBool(name, data, val) + err = d.decodeBool(name, input, outVal) case reflect.Interface: - err = d.decodeBasic(name, data, val) + err = d.decodeBasic(name, input, outVal) case reflect.String: - err = d.decodeString(name, data, val) + err = d.decodeString(name, input, outVal) case reflect.Int: - err = d.decodeInt(name, data, val) + err = d.decodeInt(name, input, outVal) case reflect.Uint: - err = d.decodeUint(name, data, val) + err = d.decodeUint(name, input, outVal) case reflect.Float32: - err = d.decodeFloat(name, data, val) + err = d.decodeFloat(name, input, outVal) case reflect.Struct: - err = d.decodeStruct(name, data, val) + err = d.decodeStruct(name, input, outVal) case reflect.Map: - err = d.decodeMap(name, data, val) + err = d.decodeMap(name, input, outVal) case reflect.Ptr: - err = d.decodePtr(name, data, val) + err = d.decodePtr(name, input, outVal) case reflect.Slice: - err = d.decodeSlice(name, data, val) + err = d.decodeSlice(name, input, outVal) + case reflect.Array: + err = d.decodeArray(name, input, outVal) case reflect.Func: - err = d.decodeFunc(name, data, val) + err = d.decodeFunc(name, input, outVal) default: // If we reached this point then we weren't able to decode it - return fmt.Errorf("%s: unsupported type: %s", name, dataKind) + return fmt.Errorf("%s: unsupported type: %s", name, inputKind) } // If we reached here, then we successfully decoded SOMETHING, so - // mark the key as used if we're tracking metadata. + // mark the key as used if we're tracking metainput. if d.config.Metadata != nil && name != "" { d.config.Metadata.Keys = append(d.config.Metadata.Keys, name) } @@ -251,6 +303,9 @@ func (d *Decoder) decode(name string, data interface{}, val reflect.Value) error // This decodes a basic type (bool, int, string, etc.) and sets the // value to "data" of that type. func (d *Decoder) decodeBasic(name string, data interface{}, val reflect.Value) error { + if val.IsValid() && val.Elem().IsValid() { + return d.decode(name, data, val.Elem()) + } dataVal := reflect.ValueOf(data) if !dataVal.IsValid() { dataVal = reflect.Zero(val.Type()) @@ -287,12 +342,22 @@ func (d *Decoder) decodeString(name string, data interface{}, val reflect.Value) val.SetString(strconv.FormatUint(dataVal.Uint(), 10)) case dataKind == reflect.Float32 && d.config.WeaklyTypedInput: val.SetString(strconv.FormatFloat(dataVal.Float(), 'f', -1, 64)) - case dataKind == reflect.Slice && d.config.WeaklyTypedInput: + case dataKind == reflect.Slice && d.config.WeaklyTypedInput, + dataKind == reflect.Array && d.config.WeaklyTypedInput: dataType := dataVal.Type() elemKind := dataType.Elem().Kind() - switch { - case elemKind == reflect.Uint8: - val.SetString(string(dataVal.Interface().([]uint8))) + switch elemKind { + case reflect.Uint8: + var uints []uint8 + if dataKind == reflect.Array { + uints = make([]uint8, dataVal.Len(), dataVal.Len()) + for i := range uints { + uints[i] = dataVal.Index(i).Interface().(uint8) + } + } else { + uints = dataVal.Interface().([]uint8) + } + val.SetString(string(uints)) default: converted = false } @@ -436,7 +501,7 @@ func (d *Decoder) decodeFloat(name string, data interface{}, val reflect.Value) case dataKind == reflect.Uint: val.SetFloat(float64(dataVal.Uint())) case dataKind == reflect.Float32: - val.SetFloat(float64(dataVal.Float())) + val.SetFloat(dataVal.Float()) case dataKind == reflect.Bool && d.config.WeaklyTypedInput: if dataVal.Bool() { val.SetFloat(1) @@ -482,34 +547,50 @@ func (d *Decoder) decodeMap(name string, data interface{}, val reflect.Value) er valMap = reflect.MakeMap(mapType) } - // Check input type + // Check input type and based on the input type jump to the proper func dataVal := reflect.Indirect(reflect.ValueOf(data)) - if dataVal.Kind() != reflect.Map { - // In weak mode, we accept a slice of maps as an input... + switch dataVal.Kind() { + case reflect.Map: + return d.decodeMapFromMap(name, dataVal, val, valMap) + + case reflect.Struct: + return d.decodeMapFromStruct(name, dataVal, val, valMap) + + case reflect.Array, reflect.Slice: if d.config.WeaklyTypedInput { - switch dataVal.Kind() { - case reflect.Array, reflect.Slice: - // Special case for BC reasons (covered by tests) - if dataVal.Len() == 0 { - val.Set(valMap) - return nil - } - - for i := 0; i < dataVal.Len(); i++ { - err := d.decode( - fmt.Sprintf("%s[%d]", name, i), - dataVal.Index(i).Interface(), val) - if err != nil { - return err - } - } - - return nil - } + return d.decodeMapFromSlice(name, dataVal, val, valMap) } + fallthrough + + default: return fmt.Errorf("'%s' expected a map, got '%s'", name, dataVal.Kind()) } +} + +func (d *Decoder) decodeMapFromSlice(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + // Special case for BC reasons (covered by tests) + if dataVal.Len() == 0 { + val.Set(valMap) + return nil + } + + for i := 0; i < dataVal.Len(); i++ { + err := d.decode( + fmt.Sprintf("%s[%d]", name, i), + dataVal.Index(i).Interface(), val) + if err != nil { + return err + } + } + + return nil +} + +func (d *Decoder) decodeMapFromMap(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + valType := val.Type() + valKeyType := valType.Key() + valElemType := valType.Elem() // Accumulate errors errors := make([]string, 0) @@ -546,22 +627,88 @@ func (d *Decoder) decodeMap(name string, data interface{}, val reflect.Value) er return nil } +func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val reflect.Value, valMap reflect.Value) error { + typ := dataVal.Type() + for i := 0; i < typ.NumField(); i++ { + // Get the StructField first since this is a cheap operation. If the + // field is unexported, then ignore it. + f := typ.Field(i) + if f.PkgPath != "" { + continue + } + + // Next get the actual value of this field and verify it is assignable + // to the map value. + v := dataVal.Field(i) + if !v.Type().AssignableTo(valMap.Type().Elem()) { + return fmt.Errorf("cannot assign type '%s' to map value field of type '%s'", v.Type(), valMap.Type().Elem()) + } + + // Determine the name of the key in the map + keyName := f.Name + tagValue := f.Tag.Get(d.config.TagName) + tagValue = strings.SplitN(tagValue, ",", 2)[0] + if tagValue != "" { + if tagValue == "-" { + continue + } + + keyName = tagValue + } + + switch v.Kind() { + // this is an embedded struct, so handle it differently + case reflect.Struct: + x := reflect.New(v.Type()) + x.Elem().Set(v) + + vType := valMap.Type() + vKeyType := vType.Key() + vElemType := vType.Elem() + mType := reflect.MapOf(vKeyType, vElemType) + vMap := reflect.MakeMap(mType) + + err := d.decode(keyName, x.Interface(), vMap) + if err != nil { + return err + } + + valMap.SetMapIndex(reflect.ValueOf(keyName), vMap) + + default: + valMap.SetMapIndex(reflect.ValueOf(keyName), v) + } + } + + if val.CanAddr() { + val.Set(valMap) + } + + return nil +} + func (d *Decoder) decodePtr(name string, data interface{}, val reflect.Value) error { // Create an element of the concrete (non pointer) type and decode // into that. Then set the value of the pointer to this type. valType := val.Type() valElemType := valType.Elem() - realVal := val - if realVal.IsNil() || d.config.ZeroFields { - realVal = reflect.New(valElemType) - } + if val.CanSet() { + realVal := val + if realVal.IsNil() || d.config.ZeroFields { + realVal = reflect.New(valElemType) + } - if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil { - return err - } + if err := d.decode(name, data, reflect.Indirect(realVal)); err != nil { + return err + } - val.Set(realVal) + val.Set(realVal) + } else { + if err := d.decode(name, data, reflect.Indirect(val)); err != nil { + return err + } + } return nil } @@ -597,7 +744,8 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) val.Set(reflect.MakeSlice(sliceType, 0, 0)) return nil } - + case dataValKind == reflect.String && valElemType.Kind() == reflect.Uint8: + return d.decodeSlice(name, []byte(dataVal.String()), val) // All other types we try to convert to the slice type // and "lift" it into it. i.e. a string becomes a string slice. default: @@ -605,7 +753,6 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) return d.decodeSlice(name, []interface{}{data}, val) } } - return fmt.Errorf( "'%s': source data must be an array or slice, got %s", name, dataValKind) @@ -642,6 +789,73 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) return nil } +func (d *Decoder) decodeArray(name string, data interface{}, val reflect.Value) error { + dataVal := reflect.Indirect(reflect.ValueOf(data)) + dataValKind := dataVal.Kind() + valType := val.Type() + valElemType := valType.Elem() + arrayType := reflect.ArrayOf(valType.Len(), valElemType) + + valArray := val + + if valArray.Interface() == reflect.Zero(valArray.Type()).Interface() || d.config.ZeroFields { + // Check input type + if dataValKind != reflect.Array && dataValKind != reflect.Slice { + if d.config.WeaklyTypedInput { + switch { + // Empty maps turn into empty arrays + case dataValKind == reflect.Map: + if dataVal.Len() == 0 { + val.Set(reflect.Zero(arrayType)) + return nil + } + + // All other types we try to convert to the array type + // and "lift" it into it. i.e. a string becomes a string array. + default: + // Just re-try this function with data as a slice. + return d.decodeArray(name, []interface{}{data}, val) + } + } + + return fmt.Errorf( + "'%s': source data must be an array or slice, got %s", name, dataValKind) + + } + if dataVal.Len() > arrayType.Len() { + return fmt.Errorf( + "'%s': expected source data to have length less or equal to %d, got %d", name, arrayType.Len(), dataVal.Len()) + + } + + // Make a new array to hold our result, same size as the original data. + valArray = reflect.New(arrayType).Elem() + } + + // Accumulate any errors + errors := make([]string, 0) + + for i := 0; i < dataVal.Len(); i++ { + currentData := dataVal.Index(i).Interface() + currentField := valArray.Index(i) + + fieldName := fmt.Sprintf("%s[%d]", name, i) + if err := d.decode(fieldName, currentData, currentField); err != nil { + errors = appendErrors(errors, err) + } + } + + // Finally, set the value to the array we built up + val.Set(valArray) + + // If there were errors, we return those + if len(errors) > 0 { + return &Error{errors} + } + + return nil +} + func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) error { dataVal := reflect.Indirect(reflect.ValueOf(data)) @@ -681,7 +895,11 @@ func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) // Compile the list of all the fields that we're going to be decoding // from all the structs. - fields := make(map[*reflect.StructField]reflect.Value) + type field struct { + field reflect.StructField + val reflect.Value + } + fields := []field{} for len(structs) > 0 { structVal := structs[0] structs = structs[1:] @@ -707,20 +925,22 @@ func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) errors = appendErrors(errors, fmt.Errorf("%s: unsupported type for squash: %s", fieldType.Name, fieldKind)) } else { - structs = append(structs, val.FieldByName(fieldType.Name)) + structs = append(structs, structVal.FieldByName(fieldType.Name)) } continue } // Normal struct field, store it away - fields[&fieldType] = structVal.Field(i) + fields = append(fields, field{fieldType, structVal.Field(i)}) } } - for fieldType, field := range fields { - fieldName := fieldType.Name + // for fieldType, field := range fields { + for _, f := range fields { + field, fieldValue := f.field, f.val + fieldName := field.Name - tagValue := fieldType.Tag.Get(d.config.TagName) + tagValue := field.Tag.Get(d.config.TagName) tagValue = strings.SplitN(tagValue, ",", 2)[0] if tagValue != "" { fieldName = tagValue @@ -755,14 +975,14 @@ func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) // Delete the key we're using from the unused map so we stop tracking delete(dataValKeysUnused, rawMapKey.Interface()) - if !field.IsValid() { + if !fieldValue.IsValid() { // This should never happen panic("field is not valid") } // If we can't set the field, then it is unexported or something, // and we just continue onwards. - if !field.CanSet() { + if !fieldValue.CanSet() { continue } @@ -772,7 +992,7 @@ func (d *Decoder) decodeStruct(name string, data interface{}, val reflect.Value) fieldName = fmt.Sprintf("%s.%s", name, fieldName) } - if err := d.decode(fieldName, rawMapVal.Interface(), field); err != nil { + if err := d.decode(fieldName, rawMapVal.Interface(), fieldValue); err != nil { errors = appendErrors(errors, err) } } diff --git a/vendor/github.com/mschoch/smat/LICENSE b/vendor/github.com/mschoch/smat/LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/vendor/github.com/mschoch/smat/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/mschoch/smat/README.md b/vendor/github.com/mschoch/smat/README.md new file mode 100644 index 00000000..f5ca1c54 --- /dev/null +++ b/vendor/github.com/mschoch/smat/README.md @@ -0,0 +1,166 @@ +# smat – State Machine Assisted Testing + +The concept is simple, describe valid uses of your library as states and actions. States describe which actions are possible, and with what probability they should occur. Actions mutate the context and transition to another state. + +By doing this, two things are possible: + +1. Use [go-fuzz](https://github.com/dvyukov/go-fuzz) to find/test interesting sequences of operations on your library. + +2. Automate longevity testing of your application by performing long sequences of valid operations. + +**NOTE**: both of these can also incorporate validation logic (not just failure detection by building validation into the state machine) + +## Status + +The API is still not stable. This is brand new and we'll probably change things we don't like... + +[![Build Status](https://travis-ci.org/mschoch/smat.svg?branch=master)](https://travis-ci.org/mschoch/smat) +[![Coverage Status](https://coveralls.io/repos/github/mschoch/smat/badge.svg?branch=master)](https://coveralls.io/github/mschoch/smat?branch=master) +[![GoDoc](https://godoc.org/github.com/mschoch/smat?status.svg)](https://godoc.org/github.com/mschoch/smat) +[![codebeat badge](https://codebeat.co/badges/c3ff6180-a241-4128-97f0-fa6bf6f48752)](https://codebeat.co/projects/github-com-mschoch-smat) +[![Go Report Card](https://goreportcard.com/badge/github.com/mschoch/smat)](https://goreportcard.com/report/github.com/mschoch/smat) + +## License + +Apache 2.0 + +## How do I use it? + +### smat.Context + +Choose a structure to keep track of any state. You pass in an instance of this when you start, and it will be passed to every action when it executes. The actions may mutate this context. + +For example, consider a database library, once you open a database handle, you need to use it inside of the other actions. So you might use a structure like: + +``` +type context struct { + db *DB +} +``` + +### smat.State + +A state represents a state that your application/library can be in, and the probabilities thats certain actions should be taken. + +For example, consider a database library, in a state where the database is open, there many things you can do. Let's consider just two right now, you can set a value, or you can delete a value. + +``` +func dbOpen(next byte) smat.ActionID { + return smat.PercentExecute(next, + smat.PercentAction{50, setValue}, + smat.PercentAction{50, deleteValue}, + ) +} +``` + +This says that in the open state, there are two valid actions, 50% of the time you should set a value and 50% of the time you should delete a value. **NOTE**: these percentages are just for characterizing the test workload. + +### smat.Action + +Actions are functions that do some work, optionally mutate the context, and indicate the next state to transition to. Below we see an example action to set value in a database. + +``` +func setValueFunc(ctx smat.Context) (next smat.State, err error) { + // type assert to our custom context type + context := ctx.(*context) + // perform the operation + err = context.db.Set("k", "v") + if err != nil { + return nil, err + } + // return the new state + return dbOpen, nil +} +``` + +### smat.ActionID and smat.ActionMap + +Actions are just functions, and since we can't compare functions in Go, we need to introduce an external identifier for them. This allows us to build a bi-directional mapping which we'll take advantage of later. + +``` +const ( + setup smat.ActionID = iota + teardown + setValue + deleteValue +) + +var actionMap = smat.ActionMap{ + setup: setupFunc, + teardown: teardownFunc, + setValue: setValueFunc, + deleteValue: deleteValueFunc, +} +``` + +### smat.ActionSeq + +A common way that many users think about a library is as a sequence of actions to be performed. Using the ActionID's that we've already seen we can build up sequences of operations. + +``` + actionSeq := smat.ActionSeq{ + open, + setValue, + setValue, + setValue, + } +``` + +Notice that we build these actions using the constants we defined above, and because of this we can have a bi-directional mapping between a stream of bytes (driving the state machine) and a sequence of actions to be performed. + +## Fuzzing + +We've built a lot of pieces, lets wire it up to go-fuzz. + +``` +func Fuzz(data []byte) int { + return smat.Fuzz(&context{}, setup, teardown, actionMap, data) +} +``` + +* The first argument is an instance of context structure. +* The second argument is the ActionID of our setup function. The setup function does not consume any of the input stream and is used to initialize the context and determine the start state. +* The third argument is the teardown function. This will be called unconditionally to clean up any resources associated with the test. +* The fourth argument is the actionMap which maps all ActionIDs to Actions. +* The fifth argument is the data passed in from the go-fuzz application. + +### Generating Initial go-fuzz Corpus + +Earlier we mentioned the bi-directional mapping between Actions and the byte stream driving the state machine. We can now leverage this to build the inital go-fuzz corpus. + +Using the `ActinSeq`s we learned about earlier we can build up a list of them as: + + var actionSeqs = []smat.ActionSeq{...} + +Then, we can write them out to disk using: + +``` +for i, actionSeq := range actionSeqs { + byteSequence, err := actionSeq.ByteEncoding(&context{}, setup, teardown, actionMap) + if err != nil { + // handle error + } + os.MkdirAll("workdir/corpus", 0700) + ioutil.WriteFile(fmt.Sprintf("workdir/corpus/%d", i), byteSequence, 0600) +} +``` + +You can then either put this into a test case or a main application depending on your needs. + +## Longevity Testing + +Fuzzing is great, but most of your corpus is likely to be shorter meaningful sequences. And go-fuzz works to find shortest sequences that cause problems, but sometimes you actually want to explore longer sequences that appear to go-fuzz as not triggering additional code coverage. + +For these cases we have another helper you can use: + +``` + Longevity(ctx, setup, teardown, actionMap, 0, closeChan) +``` + +The first four arguments are the same, the last two are: +* random seed used to ensure repeatable tests +* closeChan (chan struct{}) - close this channel if you want the function to stop and return ErrClosed, otherwise it will run forever + +## Examples + +See the examples directory for a working example that tests some BoltDB functionality. diff --git a/vendor/github.com/mschoch/smat/actionseq.go b/vendor/github.com/mschoch/smat/actionseq.go new file mode 100644 index 00000000..6c8297f8 --- /dev/null +++ b/vendor/github.com/mschoch/smat/actionseq.go @@ -0,0 +1,61 @@ +// Copyright (c) 2016 Marty Schoch + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an "AS +// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language +// governing permissions and limitations under the License. + +package smat + +// ActionSeq represents a sequence of actions, used for populating a corpus +// of byte sequences for the corresponding fuzz tests +type ActionSeq []ActionID + +// ByteEncoding runs the FSM to produce a byte sequence to trigger the +// desired action +func (a ActionSeq) ByteEncoding(ctx Context, setup, teardown ActionID, actionMap ActionMap) ([]byte, error) { + setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown) + if err != nil { + return nil, err + } + state, err := setupFunc(ctx) + if err != nil { + return nil, err + } + defer func() { + _, _ = teardownFunc(ctx) + }() + + var rv []byte + for _, actionID := range a { + b, err := probeStateForAction(state, actionID) + if err != nil { + return nil, err + } + rv = append(rv, b) + action, ok := actionMap[actionID] + if !ok { + continue + } + state, err = action(ctx) + if err != nil { + return nil, err + } + } + return rv, nil +} + +func probeStateForAction(state State, actionID ActionID) (byte, error) { + for i := 0; i < 256; i++ { + nextActionID := state(byte(i)) + if nextActionID == actionID { + return byte(i), nil + } + } + return 0, ErrActionNotPossible +} diff --git a/vendor/github.com/mschoch/smat/smat.go b/vendor/github.com/mschoch/smat/smat.go new file mode 100644 index 00000000..f6ea4975 --- /dev/null +++ b/vendor/github.com/mschoch/smat/smat.go @@ -0,0 +1,161 @@ +// Copyright (c) 2016 Marty Schoch + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the +// License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an "AS +// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language +// governing permissions and limitations under the License. + +package smat + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" +) + +// Logger is a configurable logger used by this package +// by default output is discarded +var Logger = log.New(ioutil.Discard, "smat ", log.LstdFlags) + +// Context is a container for any user state +type Context interface{} + +// State is a function which describes which action to perform in the event +// that a particular byte is seen +type State func(next byte) ActionID + +// PercentAction describes the frequency with which an action should occur +// for example: Action{Percent:10, Action:DonateMoney} means that 10% of +// the time you should donate money. +type PercentAction struct { + Percent int + Action ActionID +} + +// Action is any function which returns the next state to transition to +// it can optionally mutate the provided context object +// if any error occurs, it may return an error which will abort execution +type Action func(Context) (State, error) + +// ActionID is a unique identifier for an action +type ActionID int + +// NopAction does nothing and simply continues to the next input +var NopAction ActionID = -1 + +// ActionMap is a mapping form ActionID to Action +type ActionMap map[ActionID]Action + +func (a ActionMap) findSetupTeardown(setup, teardown ActionID) (Action, Action, error) { + setupFunc, ok := a[setup] + if !ok { + return nil, nil, ErrSetupMissing + } + teardownFunc, ok := a[teardown] + if !ok { + return nil, nil, ErrTeardownMissing + } + return setupFunc, teardownFunc, nil +} + +// Fuzz runs the fuzzing state machine with the provided context +// first, the setup action is executed unconditionally +// the start state is determined by this action +// actionMap is a lookup table for all actions +// the data byte slice determines all future state transitions +// finally, the teardown action is executed unconditionally for cleanup +func Fuzz(ctx Context, setup, teardown ActionID, actionMap ActionMap, data []byte) int { + reader := bytes.NewReader(data) + err := runReader(ctx, setup, teardown, actionMap, reader, nil) + if err != nil { + panic(err) + } + return 1 +} + +// Longevity runs the state machine with the provided context +// first, the setup action is executed unconditionally +// the start state is determined by this action +// actionMap is a lookup table for all actions +// random bytes are generated to determine all future state transitions +// finally, the teardown action is executed unconditionally for cleanup +func Longevity(ctx Context, setup, teardown ActionID, actionMap ActionMap, seed int64, closeChan chan struct{}) error { + source := rand.NewSource(seed) + return runReader(ctx, setup, teardown, actionMap, rand.New(source), closeChan) +} + +var ( + // ErrSetupMissing is returned when the setup action cannot be found + ErrSetupMissing = fmt.Errorf("setup action missing") + // ErrTeardownMissing is returned when the teardown action cannot be found + ErrTeardownMissing = fmt.Errorf("teardown action missing") + // ErrClosed is returned when the closeChan was closed to cancel the op + ErrClosed = fmt.Errorf("closed") + // ErrActionNotPossible is returned when an action is encountered in a + // FuzzCase that is not possible in the current state + ErrActionNotPossible = fmt.Errorf("action not possible in state") +) + +func runReader(ctx Context, setup, teardown ActionID, actionMap ActionMap, r io.Reader, closeChan chan struct{}) error { + setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown) + if err != nil { + return err + } + Logger.Printf("invoking setup action") + state, err := setupFunc(ctx) + if err != nil { + return err + } + defer func() { + Logger.Printf("invoking teardown action") + _, _ = teardownFunc(ctx) + }() + + reader := bufio.NewReader(r) + for next, err := reader.ReadByte(); err == nil; next, err = reader.ReadByte() { + select { + case <-closeChan: + return ErrClosed + default: + actionID := state(next) + action, ok := actionMap[actionID] + if !ok { + Logger.Printf("no such action defined, continuing") + continue + } + Logger.Printf("invoking action - %d", actionID) + state, err = action(ctx) + if err != nil { + Logger.Printf("it was action %d that returned err %v", actionID, err) + return err + } + } + } + return err +} + +// PercentExecute interprets the next byte as a random value and normalizes it +// to values 0-99, it then looks to see which action should be execued based +// on the action distributions +func PercentExecute(next byte, pas ...PercentAction) ActionID { + percent := int(99 * int(next) / 255) + + sofar := 0 + for _, pa := range pas { + sofar = sofar + pa.Percent + if percent < sofar { + return pa.Action + } + + } + return NopAction +} diff --git a/vendor/github.com/pelletier/go-buffruneio/README.md b/vendor/github.com/pelletier/go-buffruneio/README.md deleted file mode 100644 index ff608b3a..00000000 --- a/vendor/github.com/pelletier/go-buffruneio/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# buffruneio - -[![Tests Status](https://travis-ci.org/pelletier/go-buffruneio.svg?branch=master)](https://travis-ci.org/pelletier/go-buffruneio) -[![GoDoc](https://godoc.org/github.com/pelletier/go-buffruneio?status.svg)](https://godoc.org/github.com/pelletier/go-buffruneio) - -Buffruneio is a wrapper around bufio to provide buffered runes access with -unlimited unreads. - -```go -import "github.com/pelletier/go-buffruneio" -``` - -## Examples - -```go -import ( - "fmt" - "github.com/pelletier/go-buffruneio" - "strings" -) - -reader := buffruneio.NewReader(strings.NewReader("abcd")) -fmt.Println(reader.ReadRune()) // 'a' -fmt.Println(reader.ReadRune()) // 'b' -fmt.Println(reader.ReadRune()) // 'c' -reader.UnreadRune() -reader.UnreadRune() -fmt.Println(reader.ReadRune()) // 'b' -fmt.Println(reader.ReadRune()) // 'c' -``` - -## Documentation - -The documentation and additional examples are available at -[godoc.org](http://godoc.org/github.com/pelletier/go-buffruneio). - -## Contribute - -Feel free to report bugs and patches using GitHub's pull requests system on -[pelletier/go-toml](https://github.com/pelletier/go-buffruneio). Any feedback is -much appreciated! - -## LICENSE - -Copyright (c) 2016 Thomas Pelletier - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/pelletier/go-buffruneio/buffruneio.go b/vendor/github.com/pelletier/go-buffruneio/buffruneio.go deleted file mode 100644 index 4e6d6ea6..00000000 --- a/vendor/github.com/pelletier/go-buffruneio/buffruneio.go +++ /dev/null @@ -1,117 +0,0 @@ -// Package buffruneio is a wrapper around bufio to provide buffered runes access with unlimited unreads. -package buffruneio - -import ( - "bufio" - "container/list" - "errors" - "io" -) - -// Rune to indicate end of file. -const ( - EOF = -(iota + 1) -) - -// ErrNoRuneToUnread is returned by UnreadRune() when the read index is already at the beginning of the buffer. -var ErrNoRuneToUnread = errors.New("no rune to unwind") - -// Reader implements runes buffering for an io.Reader object. -type Reader struct { - buffer *list.List - current *list.Element - input *bufio.Reader -} - -// NewReader returns a new Reader. -func NewReader(rd io.Reader) *Reader { - return &Reader{ - buffer: list.New(), - input: bufio.NewReader(rd), - } -} - -type runeWithSize struct { - r rune - size int -} - -func (rd *Reader) feedBuffer() error { - r, size, err := rd.input.ReadRune() - - if err != nil { - if err != io.EOF { - return err - } - r = EOF - } - - newRuneWithSize := runeWithSize{r, size} - - rd.buffer.PushBack(newRuneWithSize) - if rd.current == nil { - rd.current = rd.buffer.Back() - } - return nil -} - -// ReadRune reads the next rune from buffer, or from the underlying reader if needed. -func (rd *Reader) ReadRune() (rune, int, error) { - if rd.current == rd.buffer.Back() || rd.current == nil { - err := rd.feedBuffer() - if err != nil { - return EOF, 0, err - } - } - - runeWithSize := rd.current.Value.(runeWithSize) - rd.current = rd.current.Next() - return runeWithSize.r, runeWithSize.size, nil -} - -// UnreadRune pushes back the previously read rune in the buffer, extending it if needed. -func (rd *Reader) UnreadRune() error { - if rd.current == rd.buffer.Front() { - return ErrNoRuneToUnread - } - if rd.current == nil { - rd.current = rd.buffer.Back() - } else { - rd.current = rd.current.Prev() - } - return nil -} - -// Forget removes runes stored before the current stream position index. -func (rd *Reader) Forget() { - if rd.current == nil { - rd.current = rd.buffer.Back() - } - for ; rd.current != rd.buffer.Front(); rd.buffer.Remove(rd.current.Prev()) { - } -} - -// PeekRune returns at most the next n runes, reading from the uderlying source if -// needed. Does not move the current index. It includes EOF if reached. -func (rd *Reader) PeekRunes(n int) []rune { - res := make([]rune, 0, n) - cursor := rd.current - for i := 0; i < n; i++ { - if cursor == nil { - err := rd.feedBuffer() - if err != nil { - return res - } - cursor = rd.buffer.Back() - } - if cursor != nil { - r := cursor.Value.(runeWithSize).r - res = append(res, r) - if r == EOF { - return res - } - cursor = cursor.Next() - } - } - return res -} diff --git a/vendor/github.com/pelletier/go-toml/README.md b/vendor/github.com/pelletier/go-toml/README.md index b8137e02..0d357acf 100644 --- a/vendor/github.com/pelletier/go-toml/README.md +++ b/vendor/github.com/pelletier/go-toml/README.md @@ -16,65 +16,64 @@ This library supports TOML version Go-toml provides the following features for using data parsed from TOML documents: * Load TOML documents from files and string data -* Easily navigate TOML structure using TomlTree +* Easily navigate TOML structure using Tree +* Mashaling and unmarshaling to and from data structures * Line & column position data for all parsed elements -* Query support similar to JSON-Path +* [Query support similar to JSON-Path](query/) * Syntax errors contain line and column numbers -Go-toml is designed to help cover use-cases not covered by reflection-based TOML parsing: - -* Semantic evaluation of parsed TOML -* Informing a user of mistakes in the source document, after it has been parsed -* Programatic handling of default values on a case-by-case basis -* Using a TOML document as a flexible data-store - ## Import - import "github.com/pelletier/go-toml" - -## Usage - -### Example - -Say you have a TOML file that looks like this: - -```toml -[postgres] -user = "pelletier" -password = "mypassword" +```go +import "github.com/pelletier/go-toml" ``` -Read the username and password like this: +## Usage example + +Read a TOML document: ```go -import ( - "fmt" - "github.com/pelletier/go-toml" -) +config, _ := toml.Load(` +[postgres] +user = "pelletier" +password = "mypassword"`) +// retrieve data directly +user := config.Get("postgres.user").(string) -config, err := toml.LoadFile("config.toml") -if err != nil { - fmt.Println("Error ", err.Error()) -} else { - // retrieve data directly - user := config.Get("postgres.user").(string) - password := config.Get("postgres.password").(string) +// or using an intermediate object +postgresConfig := config.Get("postgres").(*toml.Tree) +password := postgresConfig.Get("password").(string) +``` - // or using an intermediate object - configTree := config.Get("postgres").(*toml.TomlTree) - user = configTree.Get("user").(string) - password = configTree.Get("password").(string) - fmt.Println("User is ", user, ". Password is ", password) +Or use Unmarshal: - // show where elements are in the file - fmt.Println("User position: %v", configTree.GetPosition("user")) - fmt.Println("Password position: %v", configTree.GetPosition("password")) +```go +type Postgres struct { + User string + Password string +} +type Config struct { + Postgres Postgres +} - // use a query to gather elements without walking the tree - results, _ := config.Query("$..[user,password]") - for ii, item := range results.Values() { - fmt.Println("Query result %d: %v", ii, item) - } +doc := []byte(` +[Postgres] +User = "pelletier" +Password = "mypassword"`) + +config := Config{} +toml.Unmarshal(doc, &config) +fmt.Println("user=", config.Postgres.User) +``` + +Or use a query: + +```go +// use a query to gather elements without walking the tree +q, _ := query.Compile("$..[user,password]") +results := q.Execute(config) +for ii, item := range results.Values() { + fmt.Println("Query result %d: %v", ii, item) } ``` @@ -115,6 +114,18 @@ You have to make sure two kind of tests run: You can run both of them using `./test.sh`. +### Fuzzing + +The script `./fuzz.sh` is available to +run [go-fuzz](https://github.com/dvyukov/go-fuzz) on go-toml. + +## Versioning + +Go-toml follows [Semantic Versioning](http://semver.org/). The supported version +of [TOML](https://github.com/toml-lang/toml) is indicated at the beginning of +this document. The last two major versions of Go are supported +(see [Go Release Policy](https://golang.org/doc/devel/release.html#policy)). + ## License The MIT License (MIT). Read [LICENSE](LICENSE). diff --git a/vendor/github.com/pelletier/go-toml/benchmark.json b/vendor/github.com/pelletier/go-toml/benchmark.json new file mode 100644 index 00000000..86f99c6a --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.json @@ -0,0 +1,164 @@ +{ + "array": { + "key1": [ + 1, + 2, + 3 + ], + "key2": [ + "red", + "yellow", + "green" + ], + "key3": [ + [ + 1, + 2 + ], + [ + 3, + 4, + 5 + ] + ], + "key4": [ + [ + 1, + 2 + ], + [ + "a", + "b", + "c" + ] + ], + "key5": [ + 1, + 2, + 3 + ], + "key6": [ + 1, + 2 + ] + }, + "boolean": { + "False": false, + "True": true + }, + "datetime": { + "key1": "1979-05-27T07:32:00Z", + "key2": "1979-05-27T00:32:00-07:00", + "key3": "1979-05-27T00:32:00.999999-07:00" + }, + "float": { + "both": { + "key": 6.626e-34 + }, + "exponent": { + "key1": 5e+22, + "key2": 1000000, + "key3": -0.02 + }, + "fractional": { + "key1": 1, + "key2": 3.1415, + "key3": -0.01 + }, + "underscores": { + "key1": 9224617.445991227, + "key2": 1e+100 + } + }, + "fruit": [{ + "name": "apple", + "physical": { + "color": "red", + "shape": "round" + }, + "variety": [{ + "name": "red delicious" + }, + { + "name": "granny smith" + } + ] + }, + { + "name": "banana", + "variety": [{ + "name": "plantain" + }] + } + ], + "integer": { + "key1": 99, + "key2": 42, + "key3": 0, + "key4": -17, + "underscores": { + "key1": 1000, + "key2": 5349221, + "key3": 12345 + } + }, + "products": [{ + "name": "Hammer", + "sku": 738594937 + }, + {}, + { + "color": "gray", + "name": "Nail", + "sku": 284758393 + } + ], + "string": { + "basic": { + "basic": "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF." + }, + "literal": { + "multiline": { + "lines": "The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n", + "regex2": "I [dw]on't need \\d{2} apples" + }, + "quoted": "Tom \"Dubs\" Preston-Werner", + "regex": "\u003c\\i\\c*\\s*\u003e", + "winpath": "C:\\Users\\nodejs\\templates", + "winpath2": "\\\\ServerX\\admin$\\system32\\" + }, + "multiline": { + "continued": { + "key1": "The quick brown fox jumps over the lazy dog.", + "key2": "The quick brown fox jumps over the lazy dog.", + "key3": "The quick brown fox jumps over the lazy dog." + }, + "key1": "One\nTwo", + "key2": "One\nTwo", + "key3": "One\nTwo" + } + }, + "table": { + "inline": { + "name": { + "first": "Tom", + "last": "Preston-Werner" + }, + "point": { + "x": 1, + "y": 2 + } + }, + "key": "value", + "subtable": { + "key": "another value" + } + }, + "x": { + "y": { + "z": { + "w": {} + } + } + } +} diff --git a/vendor/github.com/pelletier/go-toml/benchmark.sh b/vendor/github.com/pelletier/go-toml/benchmark.sh new file mode 100755 index 00000000..8b8bb528 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +reference_ref=${1:-master} +reference_git=${2:-.} + +if ! `hash benchstat 2>/dev/null`; then + echo "Installing benchstat" + go get golang.org/x/perf/cmd/benchstat + go install golang.org/x/perf/cmd/benchstat +fi + +tempdir=`mktemp -d /tmp/go-toml-benchmark-XXXXXX` +ref_tempdir="${tempdir}/ref" +ref_benchmark="${ref_tempdir}/benchmark-`echo -n ${reference_ref}|tr -s '/' '-'`.txt" +local_benchmark="`pwd`/benchmark-local.txt" + +echo "=== ${reference_ref} (${ref_tempdir})" +git clone ${reference_git} ${ref_tempdir} >/dev/null 2>/dev/null +pushd ${ref_tempdir} >/dev/null +git checkout ${reference_ref} >/dev/null 2>/dev/null +go test -bench=. -benchmem | tee ${ref_benchmark} +popd >/dev/null + +echo "" +echo "=== local" +go test -bench=. -benchmem | tee ${local_benchmark} + +echo "" +echo "=== diff" +benchstat -delta-test=none ${ref_benchmark} ${local_benchmark} \ No newline at end of file diff --git a/vendor/github.com/pelletier/go-toml/benchmark.toml b/vendor/github.com/pelletier/go-toml/benchmark.toml new file mode 100644 index 00000000..dfd77e09 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.toml @@ -0,0 +1,244 @@ +################################################################################ +## Comment + +# Speak your mind with the hash symbol. They go from the symbol to the end of +# the line. + + +################################################################################ +## Table + +# Tables (also known as hash tables or dictionaries) are collections of +# key/value pairs. They appear in square brackets on a line by themselves. + +[table] + +key = "value" # Yeah, you can do this. + +# Nested tables are denoted by table names with dots in them. Name your tables +# whatever crap you please, just don't use #, ., [ or ]. + +[table.subtable] + +key = "another value" + +# You don't need to specify all the super-tables if you don't want to. TOML +# knows how to do it for you. + +# [x] you +# [x.y] don't +# [x.y.z] need these +[x.y.z.w] # for this to work + + +################################################################################ +## Inline Table + +# Inline tables provide a more compact syntax for expressing tables. They are +# especially useful for grouped data that can otherwise quickly become verbose. +# Inline tables are enclosed in curly braces `{` and `}`. No newlines are +# allowed between the curly braces unless they are valid within a value. + +[table.inline] + +name = { first = "Tom", last = "Preston-Werner" } +point = { x = 1, y = 2 } + + +################################################################################ +## String + +# There are four ways to express strings: basic, multi-line basic, literal, and +# multi-line literal. All strings must contain only valid UTF-8 characters. + +[string.basic] + +basic = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + +[string.multiline] + +# The following strings are byte-for-byte equivalent: +key1 = "One\nTwo" +key2 = """One\nTwo""" +key3 = """ +One +Two""" + +[string.multiline.continued] + +# The following strings are byte-for-byte equivalent: +key1 = "The quick brown fox jumps over the lazy dog." + +key2 = """ +The quick brown \ + + + fox jumps over \ + the lazy dog.""" + +key3 = """\ + The quick brown \ + fox jumps over \ + the lazy dog.\ + """ + +[string.literal] + +# What you see is what you get. +winpath = 'C:\Users\nodejs\templates' +winpath2 = '\\ServerX\admin$\system32\' +quoted = 'Tom "Dubs" Preston-Werner' +regex = '<\i\c*\s*>' + + +[string.literal.multiline] + +regex2 = '''I [dw]on't need \d{2} apples''' +lines = ''' +The first newline is +trimmed in raw strings. + All other whitespace + is preserved. +''' + + +################################################################################ +## Integer + +# Integers are whole numbers. Positive numbers may be prefixed with a plus sign. +# Negative numbers are prefixed with a minus sign. + +[integer] + +key1 = +99 +key2 = 42 +key3 = 0 +key4 = -17 + +[integer.underscores] + +# For large numbers, you may use underscores to enhance readability. Each +# underscore must be surrounded by at least one digit. +key1 = 1_000 +key2 = 5_349_221 +key3 = 1_2_3_4_5 # valid but inadvisable + + +################################################################################ +## Float + +# A float consists of an integer part (which may be prefixed with a plus or +# minus sign) followed by a fractional part and/or an exponent part. + +[float.fractional] + +key1 = +1.0 +key2 = 3.1415 +key3 = -0.01 + +[float.exponent] + +key1 = 5e+22 +key2 = 1e6 +key3 = -2E-2 + +[float.both] + +key = 6.626e-34 + +[float.underscores] + +key1 = 9_224_617.445_991_228_313 +key2 = 1e1_00 + + +################################################################################ +## Boolean + +# Booleans are just the tokens you're used to. Always lowercase. + +[boolean] + +True = true +False = false + + +################################################################################ +## Datetime + +# Datetimes are RFC 3339 dates. + +[datetime] + +key1 = 1979-05-27T07:32:00Z +key2 = 1979-05-27T00:32:00-07:00 +key3 = 1979-05-27T00:32:00.999999-07:00 + + +################################################################################ +## Array + +# Arrays are square brackets with other primitives inside. Whitespace is +# ignored. Elements are separated by commas. Data types may not be mixed. + +[array] + +key1 = [ 1, 2, 3 ] +key2 = [ "red", "yellow", "green" ] +key3 = [ [ 1, 2 ], [3, 4, 5] ] +#key4 = [ [ 1, 2 ], ["a", "b", "c"] ] # this is ok + +# Arrays can also be multiline. So in addition to ignoring whitespace, arrays +# also ignore newlines between the brackets. Terminating commas are ok before +# the closing bracket. + +key5 = [ + 1, 2, 3 +] +key6 = [ + 1, + 2, # this is ok +] + + +################################################################################ +## Array of Tables + +# These can be expressed by using a table name in double brackets. Each table +# with the same double bracketed name will be an element in the array. The +# tables are inserted in the order encountered. + +[[products]] + +name = "Hammer" +sku = 738594937 + +[[products]] + +[[products]] + +name = "Nail" +sku = 284758393 +color = "gray" + + +# You can create nested arrays of tables as well. + +[[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + +[[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" diff --git a/vendor/github.com/pelletier/go-toml/benchmark.yml b/vendor/github.com/pelletier/go-toml/benchmark.yml new file mode 100644 index 00000000..0bd19f08 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/benchmark.yml @@ -0,0 +1,121 @@ +--- +array: + key1: + - 1 + - 2 + - 3 + key2: + - red + - yellow + - green + key3: + - - 1 + - 2 + - - 3 + - 4 + - 5 + key4: + - - 1 + - 2 + - - a + - b + - c + key5: + - 1 + - 2 + - 3 + key6: + - 1 + - 2 +boolean: + 'False': false + 'True': true +datetime: + key1: '1979-05-27T07:32:00Z' + key2: '1979-05-27T00:32:00-07:00' + key3: '1979-05-27T00:32:00.999999-07:00' +float: + both: + key: 6.626e-34 + exponent: + key1: 5.0e+22 + key2: 1000000 + key3: -0.02 + fractional: + key1: 1 + key2: 3.1415 + key3: -0.01 + underscores: + key1: 9224617.445991227 + key2: 1.0e+100 +fruit: +- name: apple + physical: + color: red + shape: round + variety: + - name: red delicious + - name: granny smith +- name: banana + variety: + - name: plantain +integer: + key1: 99 + key2: 42 + key3: 0 + key4: -17 + underscores: + key1: 1000 + key2: 5349221 + key3: 12345 +products: +- name: Hammer + sku: 738594937 +- {} +- color: gray + name: Nail + sku: 284758393 +string: + basic: + basic: "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF." + literal: + multiline: + lines: | + The first newline is + trimmed in raw strings. + All other whitespace + is preserved. + regex2: I [dw]on't need \d{2} apples + quoted: Tom "Dubs" Preston-Werner + regex: "<\\i\\c*\\s*>" + winpath: C:\Users\nodejs\templates + winpath2: "\\\\ServerX\\admin$\\system32\\" + multiline: + continued: + key1: The quick brown fox jumps over the lazy dog. + key2: The quick brown fox jumps over the lazy dog. + key3: The quick brown fox jumps over the lazy dog. + key1: |- + One + Two + key2: |- + One + Two + key3: |- + One + Two +table: + inline: + name: + first: Tom + last: Preston-Werner + point: + x: 1 + y: 2 + key: value + subtable: + key: another value +x: + y: + z: + w: {} diff --git a/vendor/github.com/pelletier/go-toml/clean.sh b/vendor/github.com/pelletier/go-toml/clean.sh deleted file mode 100755 index 44d49d93..00000000 --- a/vendor/github.com/pelletier/go-toml/clean.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# fail out of the script if anything here fails -set -e - -# clear out stuff generated by test.sh -rm -rf src test_program_bin toml-test diff --git a/vendor/github.com/pelletier/go-toml/doc.go b/vendor/github.com/pelletier/go-toml/doc.go index 9156b736..d5fd98c0 100644 --- a/vendor/github.com/pelletier/go-toml/doc.go +++ b/vendor/github.com/pelletier/go-toml/doc.go @@ -1,250 +1,23 @@ -// Package toml is a TOML markup language parser. +// Package toml is a TOML parser and manipulation library. // // This version supports the specification as described in // https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md // -// TOML Parsing +// Marshaling // -// TOML data may be parsed in two ways: by file, or by string. +// Go-toml can marshal and unmarshal TOML documents from and to data +// structures. // -// // load TOML data by filename -// tree, err := toml.LoadFile("filename.toml") +// TOML document as a tree // -// // load TOML data stored in a string -// tree, err := toml.Load(stringContainingTomlData) +// Go-toml can operate on a TOML document as a tree. Use one of the Load* +// functions to parse TOML data and obtain a Tree instance, then one of its +// methods to manipulate the tree. // -// Either way, the result is a TomlTree object that can be used to navigate the -// structure and data within the original document. +// JSONPath-like queries // -// -// Getting data from the TomlTree -// -// After parsing TOML data with Load() or LoadFile(), use the Has() and Get() -// methods on the returned TomlTree, to find your way through the document data. -// -// if tree.Has("foo") { -// fmt.Println("foo is:", tree.Get("foo")) -// } -// -// Working with Paths -// -// Go-toml has support for basic dot-separated key paths on the Has(), Get(), Set() -// and GetDefault() methods. These are the same kind of key paths used within the -// TOML specification for struct tames. -// -// // looks for a key named 'baz', within struct 'bar', within struct 'foo' -// tree.Has("foo.bar.baz") -// -// // returns the key at this path, if it is there -// tree.Get("foo.bar.baz") -// -// TOML allows keys to contain '.', which can cause this syntax to be problematic -// for some documents. In such cases, use the GetPath(), HasPath(), and SetPath(), -// methods to explicitly define the path. This form is also faster, since -// it avoids having to parse the passed key for '.' delimiters. -// -// // looks for a key named 'baz', within struct 'bar', within struct 'foo' -// tree.HasPath([]string{"foo","bar","baz"}) -// -// // returns the key at this path, if it is there -// tree.GetPath([]string{"foo","bar","baz"}) -// -// Note that this is distinct from the heavyweight query syntax supported by -// TomlTree.Query() and the Query() struct (see below). -// -// Position Support -// -// Each element within the TomlTree is stored with position metadata, which is -// invaluable for providing semantic feedback to a user. This helps in -// situations where the TOML file parses correctly, but contains data that is -// not correct for the application. In such cases, an error message can be -// generated that indicates the problem line and column number in the source -// TOML document. -// -// // load TOML data -// tree, _ := toml.Load("filename.toml") -// -// // get an entry and report an error if it's the wrong type -// element := tree.Get("foo") -// if value, ok := element.(int64); !ok { -// return fmt.Errorf("%v: Element 'foo' must be an integer", tree.GetPosition("foo")) -// } -// -// // report an error if an expected element is missing -// if !tree.Has("bar") { -// return fmt.Errorf("%v: Expected 'bar' element", tree.GetPosition("")) -// } -// -// Query Support -// -// The TOML query path implementation is based loosely on the JSONPath specification: -// http://goessner.net/articles/JsonPath/ -// -// The idea behind a query path is to allow quick access to any element, or set -// of elements within TOML document, with a single expression. -// -// result, err := tree.Query("$.foo.bar.baz") -// -// This is roughly equivalent to: -// -// next := tree.Get("foo") -// if next != nil { -// next = next.Get("bar") -// if next != nil { -// next = next.Get("baz") -// } -// } -// result := next -// -// err is nil if any parsing exception occurs. -// -// If no node in the tree matches the query, result will simply contain an empty list of -// items. -// -// As illustrated above, the query path is much more efficient, especially since -// the structure of the TOML file can vary. Rather than making assumptions about -// a document's structure, a query allows the programmer to make structured -// requests into the document, and get zero or more values as a result. -// -// The syntax of a query begins with a root token, followed by any number -// sub-expressions: -// -// $ -// Root of the TOML tree. This must always come first. -// .name -// Selects child of this node, where 'name' is a TOML key -// name. -// ['name'] -// Selects child of this node, where 'name' is a string -// containing a TOML key name. -// [index] -// Selcts child array element at 'index'. -// ..expr -// Recursively selects all children, filtered by an a union, -// index, or slice expression. -// ..* -// Recursive selection of all nodes at this point in the -// tree. -// .* -// Selects all children of the current node. -// [expr,expr] -// Union operator - a logical 'or' grouping of two or more -// sub-expressions: index, key name, or filter. -// [start:end:step] -// Slice operator - selects array elements from start to -// end-1, at the given step. All three arguments are -// optional. -// [?(filter)] -// Named filter expression - the function 'filter' is -// used to filter children at this node. -// -// Query Indexes And Slices -// -// Index expressions perform no bounds checking, and will contribute no -// values to the result set if the provided index or index range is invalid. -// Negative indexes represent values from the end of the array, counting backwards. -// -// // select the last index of the array named 'foo' -// tree.Query("$.foo[-1]") -// -// Slice expressions are supported, by using ':' to separate a start/end index pair. -// -// // select up to the first five elements in the array -// tree.Query("$.foo[0:5]") -// -// Slice expressions also allow negative indexes for the start and stop -// arguments. -// -// // select all array elements. -// tree.Query("$.foo[0:-1]") -// -// Slice expressions may have an optional stride/step parameter: -// -// // select every other element -// tree.Query("$.foo[0:-1:2]") -// -// Slice start and end parameters are also optional: -// -// // these are all equivalent and select all the values in the array -// tree.Query("$.foo[:]") -// tree.Query("$.foo[0:]") -// tree.Query("$.foo[:-1]") -// tree.Query("$.foo[0:-1:]") -// tree.Query("$.foo[::1]") -// tree.Query("$.foo[0::1]") -// tree.Query("$.foo[:-1:1]") -// tree.Query("$.foo[0:-1:1]") -// -// Query Filters -// -// Query filters are used within a Union [,] or single Filter [] expression. -// A filter only allows nodes that qualify through to the next expression, -// and/or into the result set. -// -// // returns children of foo that are permitted by the 'bar' filter. -// tree.Query("$.foo[?(bar)]") -// -// There are several filters provided with the library: -// -// tree -// Allows nodes of type TomlTree. -// int -// Allows nodes of type int64. -// float -// Allows nodes of type float64. -// string -// Allows nodes of type string. -// time -// Allows nodes of type time.Time. -// bool -// Allows nodes of type bool. -// -// Query Results -// -// An executed query returns a QueryResult object. This contains the nodes -// in the TOML tree that qualify the query expression. Position information -// is also available for each value in the set. -// -// // display the results of a query -// results := tree.Query("$.foo.bar.baz") -// for idx, value := results.Values() { -// fmt.Println("%v: %v", results.Positions()[idx], value) -// } -// -// Compiled Queries -// -// Queries may be executed directly on a TomlTree object, or compiled ahead -// of time and executed discretely. The former is more convienent, but has the -// penalty of having to recompile the query expression each time. -// -// // basic query -// results := tree.Query("$.foo.bar.baz") -// -// // compiled query -// query := toml.CompileQuery("$.foo.bar.baz") -// results := query.Execute(tree) -// -// // run the compiled query again on a different tree -// moreResults := query.Execute(anotherTree) -// -// User Defined Query Filters -// -// Filter expressions may also be user defined by using the SetFilter() -// function on the Query object. The function must return true/false, which -// signifies if the passed node is kept or discarded, respectively. -// -// // create a query that references a user-defined filter -// query, _ := CompileQuery("$[?(bazOnly)]") -// -// // define the filter, and assign it to the query -// query.SetFilter("bazOnly", func(node interface{}) bool{ -// if tree, ok := node.(*TomlTree); ok { -// return tree.Has("baz") -// } -// return false // reject all other node types -// }) -// -// // run the query -// query.Execute(tree) +// The package github.com/pelletier/go-toml/query implements a system +// similar to JSONPath to quickly retrieve elements of a TOML document using a +// single expression. See the package documentation for more information. // package toml diff --git a/vendor/github.com/pelletier/go-toml/fuzz.go b/vendor/github.com/pelletier/go-toml/fuzz.go new file mode 100644 index 00000000..14570c8d --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/fuzz.go @@ -0,0 +1,31 @@ +// +build gofuzz + +package toml + +func Fuzz(data []byte) int { + tree, err := LoadBytes(data) + if err != nil { + if tree != nil { + panic("tree must be nil if there is an error") + } + return 0 + } + + str, err := tree.ToTomlString() + if err != nil { + if str != "" { + panic(`str must be "" if there is an error`) + } + panic(err) + } + + tree, err = Load(str) + if err != nil { + if tree != nil { + panic("tree must be nil if there is an error") + } + return 0 + } + + return 1 +} diff --git a/vendor/github.com/pelletier/go-toml/fuzz.sh b/vendor/github.com/pelletier/go-toml/fuzz.sh new file mode 100755 index 00000000..3204b4c4 --- /dev/null +++ b/vendor/github.com/pelletier/go-toml/fuzz.sh @@ -0,0 +1,15 @@ +#! /bin/sh +set -eu + +go get github.com/dvyukov/go-fuzz/go-fuzz +go get github.com/dvyukov/go-fuzz/go-fuzz-build + +if [ ! -e toml-fuzz.zip ]; then + go-fuzz-build github.com/pelletier/go-toml +fi + +rm -fr fuzz +mkdir -p fuzz/corpus +cp *.toml fuzz/corpus + +go-fuzz -bin=toml-fuzz.zip -workdir=fuzz diff --git a/vendor/github.com/pelletier/go-toml/keysparsing.go b/vendor/github.com/pelletier/go-toml/keysparsing.go index d62ca5fd..284db646 100644 --- a/vendor/github.com/pelletier/go-toml/keysparsing.go +++ b/vendor/github.com/pelletier/go-toml/keysparsing.go @@ -9,12 +9,14 @@ import ( "unicode" ) +// Convert the bare key group string to an array. +// The input supports double quotation to allow "." inside the key name, +// but escape sequences are not supported. Lexers must unescape them beforehand. func parseKey(key string) ([]string, error) { groups := []string{} var buffer bytes.Buffer inQuotes := false wasInQuotes := false - escapeNext := false ignoreSpace := true expectDot := false @@ -25,15 +27,7 @@ func parseKey(key string) ([]string, error) { } ignoreSpace = false } - if escapeNext { - buffer.WriteRune(char) - escapeNext = false - continue - } switch char { - case '\\': - escapeNext = true - continue case '"': if inQuotes { groups = append(groups, buffer.String()) @@ -77,9 +71,6 @@ func parseKey(key string) ([]string, error) { if inQuotes { return nil, errors.New("mismatched quotes") } - if escapeNext { - return nil, errors.New("unfinished escape sequence") - } if buffer.Len() > 0 { groups = append(groups, buffer.String()) } diff --git a/vendor/github.com/pelletier/go-toml/lexer.go b/vendor/github.com/pelletier/go-toml/lexer.go index 104f3b1f..d11de428 100644 --- a/vendor/github.com/pelletier/go-toml/lexer.go +++ b/vendor/github.com/pelletier/go-toml/lexer.go @@ -6,14 +6,12 @@ package toml import ( + "bytes" "errors" "fmt" - "io" "regexp" "strconv" "strings" - - "github.com/pelletier/go-buffruneio" ) var dateRegexp *regexp.Regexp @@ -23,29 +21,29 @@ type tomlLexStateFn func() tomlLexStateFn // Define lexer type tomlLexer struct { - input *buffruneio.Reader // Textual source - buffer []rune // Runes composing the current token - tokens chan token - depth int - line int - col int - endbufferLine int - endbufferCol int + inputIdx int + input []rune // Textual source + currentTokenStart int + currentTokenStop int + tokens []token + depth int + line int + col int + endbufferLine int + endbufferCol int } // Basic read operations on input func (l *tomlLexer) read() rune { - r, _, err := l.input.ReadRune() - if err != nil { - panic(err) - } + r := l.peek() if r == '\n' { l.endbufferLine++ l.endbufferCol = 1 } else { l.endbufferCol++ } + l.inputIdx++ return r } @@ -53,13 +51,13 @@ func (l *tomlLexer) next() rune { r := l.read() if r != eof { - l.buffer = append(l.buffer, r) + l.currentTokenStop++ } return r } func (l *tomlLexer) ignore() { - l.buffer = make([]rune, 0) + l.currentTokenStart = l.currentTokenStop l.line = l.endbufferLine l.col = l.endbufferCol } @@ -76,49 +74,46 @@ func (l *tomlLexer) fastForward(n int) { } func (l *tomlLexer) emitWithValue(t tokenType, value string) { - l.tokens <- token{ + l.tokens = append(l.tokens, token{ Position: Position{l.line, l.col}, typ: t, val: value, - } + }) l.ignore() } func (l *tomlLexer) emit(t tokenType) { - l.emitWithValue(t, string(l.buffer)) + l.emitWithValue(t, string(l.input[l.currentTokenStart:l.currentTokenStop])) } func (l *tomlLexer) peek() rune { - r, _, err := l.input.ReadRune() - if err != nil { - panic(err) + if l.inputIdx >= len(l.input) { + return eof } - l.input.UnreadRune() - return r + return l.input[l.inputIdx] +} + +func (l *tomlLexer) peekString(size int) string { + maxIdx := len(l.input) + upperIdx := l.inputIdx + size // FIXME: potential overflow + if upperIdx > maxIdx { + upperIdx = maxIdx + } + return string(l.input[l.inputIdx:upperIdx]) } func (l *tomlLexer) follow(next string) bool { - for _, expectedRune := range next { - r, _, err := l.input.ReadRune() - defer l.input.UnreadRune() - if err != nil { - panic(err) - } - if expectedRune != r { - return false - } - } - return true + return next == l.peekString(len(next)) } // Error management func (l *tomlLexer) errorf(format string, args ...interface{}) tomlLexStateFn { - l.tokens <- token{ + l.tokens = append(l.tokens, token{ Position: Position{l.line, l.col}, typ: tokenError, val: fmt.Sprintf(format, args...), - } + }) return nil } @@ -209,6 +204,14 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { return l.lexFalse } + if l.follow("inf") { + return l.lexInf + } + + if l.follow("nan") { + return l.lexNan + } + if isSpace(next) { l.skip() continue @@ -219,7 +222,7 @@ func (l *tomlLexer) lexRvalue() tomlLexStateFn { break } - possibleDate := string(l.input.PeekRunes(35)) + possibleDate := l.peekString(35) dateMatch := dateRegexp.FindString(possibleDate) if dateMatch != "" { l.fastForward(len(dateMatch)) @@ -270,6 +273,18 @@ func (l *tomlLexer) lexFalse() tomlLexStateFn { return l.lexRvalue } +func (l *tomlLexer) lexInf() tomlLexStateFn { + l.fastForward(3) + l.emit(tokenInf) + return l.lexRvalue +} + +func (l *tomlLexer) lexNan() tomlLexStateFn { + l.fastForward(3) + l.emit(tokenNan) + return l.lexRvalue +} + func (l *tomlLexer) lexEqual() tomlLexStateFn { l.next() l.emit(tokenEqual) @@ -282,6 +297,8 @@ func (l *tomlLexer) lexComma() tomlLexStateFn { return l.lexRvalue } +// Parse the key and emits its value without escape sequences. +// bare keys, basic string keys and literal string keys are supported. func (l *tomlLexer) lexKey() tomlLexStateFn { growingString := "" @@ -292,7 +309,16 @@ func (l *tomlLexer) lexKey() tomlLexStateFn { if err != nil { return l.errorf(err.Error()) } - growingString += `"` + str + `"` + growingString += str + l.next() + continue + } else if r == '\'' { + l.next() + str, err := l.lexLiteralStringAsString(`'`, false) + if err != nil { + return l.errorf(err.Error()) + } + growingString += str l.next() continue } else if r == '\n' { @@ -532,11 +558,12 @@ func (l *tomlLexer) lexTableKey() tomlLexStateFn { return l.lexInsideTableKey } +// Parse the key till "]]", but only bare keys are supported func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn { for r := l.peek(); r != eof; r = l.peek() { switch r { case ']': - if len(l.buffer) > 0 { + if l.currentTokenStop > l.currentTokenStart { l.emit(tokenKeyGroupArray) } l.next() @@ -555,11 +582,12 @@ func (l *tomlLexer) lexInsideTableArrayKey() tomlLexStateFn { return l.errorf("unclosed table array key") } +// Parse the key till "]" but only bare keys are supported func (l *tomlLexer) lexInsideTableKey() tomlLexStateFn { for r := l.peek(); r != eof; r = l.peek() { switch r { case ']': - if len(l.buffer) > 0 { + if l.currentTokenStop > l.currentTokenStart { l.emit(tokenKeyGroup) } l.next() @@ -580,11 +608,77 @@ func (l *tomlLexer) lexRightBracket() tomlLexStateFn { return l.lexRvalue } +type validRuneFn func(r rune) bool + +func isValidHexRune(r rune) bool { + return r >= 'a' && r <= 'f' || + r >= 'A' && r <= 'F' || + r >= '0' && r <= '9' || + r == '_' +} + +func isValidOctalRune(r rune) bool { + return r >= '0' && r <= '7' || r == '_' +} + +func isValidBinaryRune(r rune) bool { + return r == '0' || r == '1' || r == '_' +} + func (l *tomlLexer) lexNumber() tomlLexStateFn { r := l.peek() + + if r == '0' { + follow := l.peekString(2) + if len(follow) == 2 { + var isValidRune validRuneFn + switch follow[1] { + case 'x': + isValidRune = isValidHexRune + case 'o': + isValidRune = isValidOctalRune + case 'b': + isValidRune = isValidBinaryRune + default: + if follow[1] >= 'a' && follow[1] <= 'z' || follow[1] >= 'A' && follow[1] <= 'Z' { + return l.errorf("unknown number base: %s. possible options are x (hex) o (octal) b (binary)", string(follow[1])) + } + } + + if isValidRune != nil { + l.next() + l.next() + digitSeen := false + for { + next := l.peek() + if !isValidRune(next) { + break + } + digitSeen = true + l.next() + } + + if !digitSeen { + return l.errorf("number needs at least one digit") + } + + l.emit(tokenInteger) + + return l.lexRvalue + } + } + } + if r == '+' || r == '-' { l.next() + if l.follow("inf") { + return l.lexInf + } + if l.follow("nan") { + return l.lexNan + } } + pointSeen := false expSeen := false digitSeen := false @@ -634,7 +728,6 @@ func (l *tomlLexer) run() { for state := l.lexVoid; state != nil; { state = state() } - close(l.tokens) } func init() { @@ -642,16 +735,16 @@ func init() { } // Entry point -func lexToml(input io.Reader) chan token { - bufferedInput := buffruneio.NewReader(input) +func lexToml(inputBytes []byte) []token { + runes := bytes.Runes(inputBytes) l := &tomlLexer{ - input: bufferedInput, - tokens: make(chan token), + input: runes, + tokens: make([]token, 0, 256), line: 1, col: 1, endbufferLine: 1, endbufferCol: 1, } - go l.run() + l.run() return l.tokens } diff --git a/vendor/github.com/pelletier/go-toml/marshal.go b/vendor/github.com/pelletier/go-toml/marshal.go index a1d70104..f584ba4e 100644 --- a/vendor/github.com/pelletier/go-toml/marshal.go +++ b/vendor/github.com/pelletier/go-toml/marshal.go @@ -1,41 +1,37 @@ package toml import ( + "bytes" "errors" "fmt" + "io" "reflect" + "strconv" "strings" "time" ) -/* -TomlTree structural types and corresponding marshal types -------------------------------------------------------------------------------- -*TomlTree (*)struct, (*)map[string]interface{} -[]*TomlTree (*)[](*)struct, (*)[](*)map[string]interface{} -[]interface{} (as interface{}) (*)[]primitive, (*)[]([]interface{}) -interface{} (*)primitive - -TomlTree primitive types and corresponding marshal types ------------------------------------------------------------ -uint64 uint, uint8-uint64, pointers to same -int64 int, int8-uint64, pointers to same -float64 float32, float64, pointers to same -string string, pointers to same -bool bool, pointers to same -time.Time time.Time{}, pointers to same -*/ - type tomlOpts struct { name string + comment string + commented bool include bool omitempty bool } +type encOpts struct { + quoteMapKeys bool + arraysOneElementPerLine bool +} + +var encOptsDefaults = encOpts{ + quoteMapKeys: false, +} + var timeType = reflect.TypeOf(time.Time{}) var marshalerType = reflect.TypeOf(new(Marshaler)).Elem() -// Check if the given marshall type maps to a TomlTree primitive +// Check if the given marshall type maps to a Tree primitive func isPrimitive(mtype reflect.Type) bool { switch mtype.Kind() { case reflect.Ptr: @@ -57,7 +53,7 @@ func isPrimitive(mtype reflect.Type) bool { } } -// Check if the given marshall type maps to a TomlTree slice +// Check if the given marshall type maps to a Tree slice func isTreeSlice(mtype reflect.Type) bool { switch mtype.Kind() { case reflect.Slice: @@ -67,7 +63,7 @@ func isTreeSlice(mtype reflect.Type) bool { } } -// Check if the given marshall type maps to a non-TomlTree slice +// Check if the given marshall type maps to a non-Tree slice func isOtherSlice(mtype reflect.Type) bool { switch mtype.Kind() { case reflect.Ptr: @@ -79,7 +75,7 @@ func isOtherSlice(mtype reflect.Type) bool { } } -// Check if the given marshall type maps to a TomlTree +// Check if the given marshall type maps to a Tree func isTree(mtype reflect.Type) bool { switch mtype.Kind() { case reflect.Map: @@ -111,11 +107,94 @@ encoder, except that there is no concept of a Marshaler interface or MarshalTOML function for sub-structs, and currently only definite types can be marshaled (i.e. no `interface{}`). +The following struct annotations are supported: + + toml:"Field" Overrides the field's name to output. + omitempty When set, empty values and groups are not emitted. + comment:"comment" Emits a # comment on the same line. This supports new lines. + commented:"true" Emits the value as commented. + Note that pointers are automatically assigned the "omitempty" option, as TOML -explicity does not handle null values (saying instead the label should be +explicitly does not handle null values (saying instead the label should be dropped). + +Tree structural types and corresponding marshal types: + + *Tree (*)struct, (*)map[string]interface{} + []*Tree (*)[](*)struct, (*)[](*)map[string]interface{} + []interface{} (as interface{}) (*)[]primitive, (*)[]([]interface{}) + interface{} (*)primitive + +Tree primitive types and corresponding marshal types: + + uint64 uint, uint8-uint64, pointers to same + int64 int, int8-uint64, pointers to same + float64 float32, float64, pointers to same + string string, pointers to same + bool bool, pointers to same + time.Time time.Time{}, pointers to same */ func Marshal(v interface{}) ([]byte, error) { + return NewEncoder(nil).marshal(v) +} + +// Encoder writes TOML values to an output stream. +type Encoder struct { + w io.Writer + encOpts +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{ + w: w, + encOpts: encOptsDefaults, + } +} + +// Encode writes the TOML encoding of v to the stream. +// +// See the documentation for Marshal for details. +func (e *Encoder) Encode(v interface{}) error { + b, err := e.marshal(v) + if err != nil { + return err + } + if _, err := e.w.Write(b); err != nil { + return err + } + return nil +} + +// QuoteMapKeys sets up the encoder to encode +// maps with string type keys with quoted TOML keys. +// +// This relieves the character limitations on map keys. +func (e *Encoder) QuoteMapKeys(v bool) *Encoder { + e.quoteMapKeys = v + return e +} + +// ArraysWithOneElementPerLine sets up the encoder to encode arrays +// with more than one element on multiple lines instead of one. +// +// For example: +// +// A = [1,2,3] +// +// Becomes +// +// A = [ +// 1, +// 2, +// 3, +// ] +func (e *Encoder) ArraysWithOneElementPerLine(v bool) *Encoder { + e.arraysOneElementPerLine = v + return e +} + +func (e *Encoder) marshal(v interface{}) ([]byte, error) { mtype := reflect.TypeOf(v) if mtype.Kind() != reflect.Struct { return []byte{}, errors.New("Only a struct can be marshaled to TOML") @@ -124,51 +203,62 @@ func Marshal(v interface{}) ([]byte, error) { if isCustomMarshaler(mtype) { return callCustomMarshaler(sval) } - t, err := valueToTree(mtype, sval) + t, err := e.valueToTree(mtype, sval) if err != nil { return []byte{}, err } - s, err := t.ToTomlString() - return []byte(s), err + + var buf bytes.Buffer + _, err = t.writeTo(&buf, "", "", 0, e.arraysOneElementPerLine) + + return buf.Bytes(), err } // Convert given marshal struct or map value to toml tree -func valueToTree(mtype reflect.Type, mval reflect.Value) (*TomlTree, error) { +func (e *Encoder) valueToTree(mtype reflect.Type, mval reflect.Value) (*Tree, error) { if mtype.Kind() == reflect.Ptr { - return valueToTree(mtype.Elem(), mval.Elem()) + return e.valueToTree(mtype.Elem(), mval.Elem()) } - tval := newTomlTree() + tval := newTree() switch mtype.Kind() { case reflect.Struct: for i := 0; i < mtype.NumField(); i++ { mtypef, mvalf := mtype.Field(i), mval.Field(i) opts := tomlOptions(mtypef) if opts.include && (!opts.omitempty || !isZero(mvalf)) { - val, err := valueToToml(mtypef.Type, mvalf) + val, err := e.valueToToml(mtypef.Type, mvalf) if err != nil { return nil, err } - tval.Set(opts.name, val) + tval.SetWithComment(opts.name, opts.comment, opts.commented, val) } } case reflect.Map: for _, key := range mval.MapKeys() { mvalf := mval.MapIndex(key) - val, err := valueToToml(mtype.Elem(), mvalf) + val, err := e.valueToToml(mtype.Elem(), mvalf) if err != nil { return nil, err } - tval.Set(key.String(), val) + if e.quoteMapKeys { + keyStr, err := tomlValueStringRepresentation(key.String(), "", e.arraysOneElementPerLine) + if err != nil { + return nil, err + } + tval.SetPath([]string{keyStr}, val) + } else { + tval.Set(key.String(), val) + } } } return tval, nil } // Convert given marshal slice to slice of Toml trees -func valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*TomlTree, error) { - tval := make([]*TomlTree, mval.Len(), mval.Len()) +func (e *Encoder) valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*Tree, error) { + tval := make([]*Tree, mval.Len(), mval.Len()) for i := 0; i < mval.Len(); i++ { - val, err := valueToTree(mtype.Elem(), mval.Index(i)) + val, err := e.valueToTree(mtype.Elem(), mval.Index(i)) if err != nil { return nil, err } @@ -178,10 +268,10 @@ func valueToTreeSlice(mtype reflect.Type, mval reflect.Value) ([]*TomlTree, erro } // Convert given marshal slice to slice of toml values -func valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) { +func (e *Encoder) valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, error) { tval := make([]interface{}, mval.Len(), mval.Len()) for i := 0; i < mval.Len(); i++ { - val, err := valueToToml(mtype.Elem(), mval.Index(i)) + val, err := e.valueToToml(mtype.Elem(), mval.Index(i)) if err != nil { return nil, err } @@ -191,19 +281,19 @@ func valueToOtherSlice(mtype reflect.Type, mval reflect.Value) (interface{}, err } // Convert given marshal value to toml value -func valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) { +func (e *Encoder) valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) { if mtype.Kind() == reflect.Ptr { - return valueToToml(mtype.Elem(), mval.Elem()) + return e.valueToToml(mtype.Elem(), mval.Elem()) } switch { case isCustomMarshaler(mtype): return callCustomMarshaler(mval) case isTree(mtype): - return valueToTree(mtype, mval) + return e.valueToTree(mtype, mval) case isTreeSlice(mtype): - return valueToTreeSlice(mtype, mval) + return e.valueToTreeSlice(mtype, mval) case isOtherSlice(mtype): - return valueToOtherSlice(mtype, mval) + return e.valueToOtherSlice(mtype, mval) default: switch mtype.Kind() { case reflect.Bool: @@ -224,25 +314,76 @@ func valueToToml(mtype reflect.Type, mval reflect.Value) (interface{}, error) { } } -/* -Unmarshal parses the TOML-encoded data and stores the result in the value -pointed to by v. Behavior is similar to the Go json encoder, except that there -is no concept of an Unmarshaler interface or UnmarshalTOML function for -sub-structs, and currently only definite types can be unmarshaled to (i.e. no -`interface{}`). -*/ +// Unmarshal attempts to unmarshal the Tree into a Go struct pointed by v. +// Neither Unmarshaler interfaces nor UnmarshalTOML functions are supported for +// sub-structs, and only definite types can be unmarshaled. +func (t *Tree) Unmarshal(v interface{}) error { + d := Decoder{tval: t} + return d.unmarshal(v) +} + +// Marshal returns the TOML encoding of Tree. +// See Marshal() documentation for types mapping table. +func (t *Tree) Marshal() ([]byte, error) { + var buf bytes.Buffer + err := NewEncoder(&buf).Encode(t) + return buf.Bytes(), err +} + +// Unmarshal parses the TOML-encoded data and stores the result in the value +// pointed to by v. Behavior is similar to the Go json encoder, except that there +// is no concept of an Unmarshaler interface or UnmarshalTOML function for +// sub-structs, and currently only definite types can be unmarshaled to (i.e. no +// `interface{}`). +// +// The following struct annotations are supported: +// +// toml:"Field" Overrides the field's name to map to. +// +// See Marshal() documentation for types mapping table. func Unmarshal(data []byte, v interface{}) error { + t, err := LoadReader(bytes.NewReader(data)) + if err != nil { + return err + } + return t.Unmarshal(v) +} + +// Decoder reads and decodes TOML values from an input stream. +type Decoder struct { + r io.Reader + tval *Tree + encOpts +} + +// NewDecoder returns a new decoder that reads from r. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{ + r: r, + encOpts: encOptsDefaults, + } +} + +// Decode reads a TOML-encoded value from it's input +// and unmarshals it in the value pointed at by v. +// +// See the documentation for Marshal for details. +func (d *Decoder) Decode(v interface{}) error { + var err error + d.tval, err = LoadReader(d.r) + if err != nil { + return err + } + return d.unmarshal(v) +} + +func (d *Decoder) unmarshal(v interface{}) error { mtype := reflect.TypeOf(v) if mtype.Kind() != reflect.Ptr || mtype.Elem().Kind() != reflect.Struct { return errors.New("Only a pointer to struct can be unmarshaled from TOML") } - t, err := Load(string(data)) - if err != nil { - return err - } - - sval, err := valueFromTree(mtype.Elem(), t) + sval, err := d.valueFromTree(mtype.Elem(), d.tval) if err != nil { return err } @@ -251,9 +392,9 @@ func Unmarshal(data []byte, v interface{}) error { } // Convert toml tree to marshal struct or map, using marshal type -func valueFromTree(mtype reflect.Type, tval *TomlTree) (reflect.Value, error) { +func (d *Decoder) valueFromTree(mtype reflect.Type, tval *Tree) (reflect.Value, error) { if mtype.Kind() == reflect.Ptr { - return unwrapPointer(mtype, tval) + return d.unwrapPointer(mtype, tval) } var mval reflect.Value switch mtype.Kind() { @@ -263,23 +404,29 @@ func valueFromTree(mtype reflect.Type, tval *TomlTree) (reflect.Value, error) { mtypef := mtype.Field(i) opts := tomlOptions(mtypef) if opts.include { - key := opts.name - exists := tval.Has(key) - if exists { + baseKey := opts.name + keysToTry := []string{baseKey, strings.ToLower(baseKey), strings.ToTitle(baseKey)} + for _, key := range keysToTry { + exists := tval.Has(key) + if !exists { + continue + } val := tval.Get(key) - mvalf, err := valueFromToml(mtypef.Type, val) + mvalf, err := d.valueFromToml(mtypef.Type, val) if err != nil { return mval, formatError(err, tval.GetPosition(key)) } mval.Field(i).Set(mvalf) + break } } } case reflect.Map: mval = reflect.MakeMap(mtype) for _, key := range tval.Keys() { - val := tval.Get(key) - mvalf, err := valueFromToml(mtype.Elem(), val) + // TODO: path splits key + val := tval.GetPath([]string{key}) + mvalf, err := d.valueFromToml(mtype.Elem(), val) if err != nil { return mval, formatError(err, tval.GetPosition(key)) } @@ -290,10 +437,10 @@ func valueFromTree(mtype reflect.Type, tval *TomlTree) (reflect.Value, error) { } // Convert toml value to marshal struct/map slice, using marshal type -func valueFromTreeSlice(mtype reflect.Type, tval []*TomlTree) (reflect.Value, error) { +func (d *Decoder) valueFromTreeSlice(mtype reflect.Type, tval []*Tree) (reflect.Value, error) { mval := reflect.MakeSlice(mtype, len(tval), len(tval)) for i := 0; i < len(tval); i++ { - val, err := valueFromTree(mtype.Elem(), tval[i]) + val, err := d.valueFromTree(mtype.Elem(), tval[i]) if err != nil { return mval, err } @@ -303,10 +450,10 @@ func valueFromTreeSlice(mtype reflect.Type, tval []*TomlTree) (reflect.Value, er } // Convert toml value to marshal primitive slice, using marshal type -func valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) { +func (d *Decoder) valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, error) { mval := reflect.MakeSlice(mtype, len(tval), len(tval)) for i := 0; i < len(tval); i++ { - val, err := valueFromToml(mtype.Elem(), tval[i]) + val, err := d.valueFromToml(mtype.Elem(), tval[i]) if err != nil { return mval, err } @@ -316,117 +463,86 @@ func valueFromOtherSlice(mtype reflect.Type, tval []interface{}) (reflect.Value, } // Convert toml value to marshal value, using marshal type -func valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) { +func (d *Decoder) valueFromToml(mtype reflect.Type, tval interface{}) (reflect.Value, error) { if mtype.Kind() == reflect.Ptr { - return unwrapPointer(mtype, tval) + return d.unwrapPointer(mtype, tval) } - switch { - case isTree(mtype): - return valueFromTree(mtype, tval.(*TomlTree)) - case isTreeSlice(mtype): - return valueFromTreeSlice(mtype, tval.([]*TomlTree)) - case isOtherSlice(mtype): - return valueFromOtherSlice(mtype, tval.([]interface{})) + + switch tval.(type) { + case *Tree: + if isTree(mtype) { + return d.valueFromTree(mtype, tval.(*Tree)) + } + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a tree", tval, tval) + case []*Tree: + if isTreeSlice(mtype) { + return d.valueFromTreeSlice(mtype, tval.([]*Tree)) + } + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to trees", tval, tval) + case []interface{}: + if isOtherSlice(mtype) { + return d.valueFromOtherSlice(mtype, tval.([]interface{})) + } + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to a slice", tval, tval) default: switch mtype.Kind() { - case reflect.Bool: - val, ok := tval.(bool) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to bool", tval, tval) + case reflect.Bool, reflect.Struct: + val := reflect.ValueOf(tval) + // if this passes for when mtype is reflect.Struct, tval is a time.Time + if !val.Type().ConvertibleTo(mtype) { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) } - return reflect.ValueOf(val), nil - case reflect.Int: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) - } - return reflect.ValueOf(int(val)), nil - case reflect.Int8: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) - } - return reflect.ValueOf(int8(val)), nil - case reflect.Int16: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) - } - return reflect.ValueOf(int16(val)), nil - case reflect.Int32: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) - } - return reflect.ValueOf(int32(val)), nil - case reflect.Int64: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to int", tval, tval) - } - return reflect.ValueOf(val), nil - case reflect.Uint: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) - } - return reflect.ValueOf(uint(val)), nil - case reflect.Uint8: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) - } - return reflect.ValueOf(uint8(val)), nil - case reflect.Uint16: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) - } - return reflect.ValueOf(uint16(val)), nil - case reflect.Uint32: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) - } - return reflect.ValueOf(uint32(val)), nil - case reflect.Uint64: - val, ok := tval.(int64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to uint", tval, tval) - } - return reflect.ValueOf(uint64(val)), nil - case reflect.Float32: - val, ok := tval.(float64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to float", tval, tval) - } - return reflect.ValueOf(float32(val)), nil - case reflect.Float64: - val, ok := tval.(float64) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to float", tval, tval) - } - return reflect.ValueOf(val), nil + + return val.Convert(mtype), nil case reflect.String: - val, ok := tval.(string) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to string", tval, tval) + val := reflect.ValueOf(tval) + // stupidly, int64 is convertible to string. So special case this. + if !val.Type().ConvertibleTo(mtype) || val.Kind() == reflect.Int64 { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) } - return reflect.ValueOf(val), nil - case reflect.Struct: - val, ok := tval.(time.Time) - if !ok { - return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to time", tval, tval) + + return val.Convert(mtype), nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + val := reflect.ValueOf(tval) + if !val.Type().ConvertibleTo(mtype) { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) } - return reflect.ValueOf(val), nil + if reflect.Indirect(reflect.New(mtype)).OverflowInt(val.Int()) { + return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String()) + } + + return val.Convert(mtype), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + val := reflect.ValueOf(tval) + if !val.Type().ConvertibleTo(mtype) { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) + } + if val.Int() < 0 { + return reflect.ValueOf(nil), fmt.Errorf("%v(%T) is negative so does not fit in %v", tval, tval, mtype.String()) + } + if reflect.Indirect(reflect.New(mtype)).OverflowUint(uint64(val.Int())) { + return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String()) + } + + return val.Convert(mtype), nil + case reflect.Float32, reflect.Float64: + val := reflect.ValueOf(tval) + if !val.Type().ConvertibleTo(mtype) { + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v", tval, tval, mtype.String()) + } + if reflect.Indirect(reflect.New(mtype)).OverflowFloat(val.Float()) { + return reflect.ValueOf(nil), fmt.Errorf("%v(%T) would overflow %v", tval, tval, mtype.String()) + } + + return val.Convert(mtype), nil default: - return reflect.ValueOf(nil), fmt.Errorf("Unmarshal can't handle %v(%v)", mtype, mtype.Kind()) + return reflect.ValueOf(nil), fmt.Errorf("Can't convert %v(%T) to %v(%v)", tval, tval, mtype, mtype.Kind()) } } } -func unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) { - val, err := valueFromToml(mtype.Elem(), tval) +func (d *Decoder) unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) { + val, err := d.valueFromToml(mtype.Elem(), tval) if err != nil { return reflect.ValueOf(nil), err } @@ -438,7 +554,12 @@ func unwrapPointer(mtype reflect.Type, tval interface{}) (reflect.Value, error) func tomlOptions(vf reflect.StructField) tomlOpts { tag := vf.Tag.Get("toml") parse := strings.Split(tag, ",") - result := tomlOpts{vf.Name, true, false} + var comment string + if c := vf.Tag.Get("comment"); c != "" { + comment = c + } + commented, _ := strconv.ParseBool(vf.Tag.Get("commented")) + result := tomlOpts{name: vf.Name, comment: comment, commented: commented, include: true, omitempty: false} if parse[0] != "" { if parse[0] == "-" && len(parse) == 1 { result.include = false diff --git a/vendor/github.com/pelletier/go-toml/match.go b/vendor/github.com/pelletier/go-toml/match.go deleted file mode 100644 index 48b0f2a1..00000000 --- a/vendor/github.com/pelletier/go-toml/match.go +++ /dev/null @@ -1,234 +0,0 @@ -package toml - -import ( - "fmt" -) - -// support function to set positions for tomlValues -// NOTE: this is done to allow ctx.lastPosition to indicate the start of any -// values returned by the query engines -func tomlValueCheck(node interface{}, ctx *queryContext) interface{} { - switch castNode := node.(type) { - case *tomlValue: - ctx.lastPosition = castNode.position - return castNode.value - case []*TomlTree: - if len(castNode) > 0 { - ctx.lastPosition = castNode[0].position - } - return node - default: - return node - } -} - -// base match -type matchBase struct { - next pathFn -} - -func (f *matchBase) setNext(next pathFn) { - f.next = next -} - -// terminating functor - gathers results -type terminatingFn struct { - // empty -} - -func newTerminatingFn() *terminatingFn { - return &terminatingFn{} -} - -func (f *terminatingFn) setNext(next pathFn) { - // do nothing -} - -func (f *terminatingFn) call(node interface{}, ctx *queryContext) { - switch castNode := node.(type) { - case *TomlTree: - ctx.result.appendResult(node, castNode.position) - case *tomlValue: - ctx.result.appendResult(node, castNode.position) - default: - // use last position for scalars - ctx.result.appendResult(node, ctx.lastPosition) - } -} - -// match single key -type matchKeyFn struct { - matchBase - Name string -} - -func newMatchKeyFn(name string) *matchKeyFn { - return &matchKeyFn{Name: name} -} - -func (f *matchKeyFn) call(node interface{}, ctx *queryContext) { - if array, ok := node.([]*TomlTree); ok { - for _, tree := range array { - item := tree.values[f.Name] - if item != nil { - f.next.call(item, ctx) - } - } - } else if tree, ok := node.(*TomlTree); ok { - item := tree.values[f.Name] - if item != nil { - f.next.call(item, ctx) - } - } -} - -// match single index -type matchIndexFn struct { - matchBase - Idx int -} - -func newMatchIndexFn(idx int) *matchIndexFn { - return &matchIndexFn{Idx: idx} -} - -func (f *matchIndexFn) call(node interface{}, ctx *queryContext) { - if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { - if f.Idx < len(arr) && f.Idx >= 0 { - f.next.call(arr[f.Idx], ctx) - } - } -} - -// filter by slicing -type matchSliceFn struct { - matchBase - Start, End, Step int -} - -func newMatchSliceFn(start, end, step int) *matchSliceFn { - return &matchSliceFn{Start: start, End: end, Step: step} -} - -func (f *matchSliceFn) call(node interface{}, ctx *queryContext) { - if arr, ok := tomlValueCheck(node, ctx).([]interface{}); ok { - // adjust indexes for negative values, reverse ordering - realStart, realEnd := f.Start, f.End - if realStart < 0 { - realStart = len(arr) + realStart - } - if realEnd < 0 { - realEnd = len(arr) + realEnd - } - if realEnd < realStart { - realEnd, realStart = realStart, realEnd // swap - } - // loop and gather - for idx := realStart; idx < realEnd; idx += f.Step { - f.next.call(arr[idx], ctx) - } - } -} - -// match anything -type matchAnyFn struct { - matchBase -} - -func newMatchAnyFn() *matchAnyFn { - return &matchAnyFn{} -} - -func (f *matchAnyFn) call(node interface{}, ctx *queryContext) { - if tree, ok := node.(*TomlTree); ok { - for _, v := range tree.values { - f.next.call(v, ctx) - } - } -} - -// filter through union -type matchUnionFn struct { - Union []pathFn -} - -func (f *matchUnionFn) setNext(next pathFn) { - for _, fn := range f.Union { - fn.setNext(next) - } -} - -func (f *matchUnionFn) call(node interface{}, ctx *queryContext) { - for _, fn := range f.Union { - fn.call(node, ctx) - } -} - -// match every single last node in the tree -type matchRecursiveFn struct { - matchBase -} - -func newMatchRecursiveFn() *matchRecursiveFn { - return &matchRecursiveFn{} -} - -func (f *matchRecursiveFn) call(node interface{}, ctx *queryContext) { - if tree, ok := node.(*TomlTree); ok { - var visit func(tree *TomlTree) - visit = func(tree *TomlTree) { - for _, v := range tree.values { - f.next.call(v, ctx) - switch node := v.(type) { - case *TomlTree: - visit(node) - case []*TomlTree: - for _, subtree := range node { - visit(subtree) - } - } - } - } - f.next.call(tree, ctx) - visit(tree) - } -} - -// match based on an externally provided functional filter -type matchFilterFn struct { - matchBase - Pos Position - Name string -} - -func newMatchFilterFn(name string, pos Position) *matchFilterFn { - return &matchFilterFn{Name: name, Pos: pos} -} - -func (f *matchFilterFn) call(node interface{}, ctx *queryContext) { - fn, ok := (*ctx.filters)[f.Name] - if !ok { - panic(fmt.Sprintf("%s: query context does not have filter '%s'", - f.Pos.String(), f.Name)) - } - switch castNode := tomlValueCheck(node, ctx).(type) { - case *TomlTree: - for _, v := range castNode.values { - if tv, ok := v.(*tomlValue); ok { - if fn(tv.value) { - f.next.call(v, ctx) - } - } else { - if fn(v) { - f.next.call(v, ctx) - } - } - } - case []interface{}: - for _, v := range castNode { - if fn(v) { - f.next.call(v, ctx) - } - } - } -} diff --git a/vendor/github.com/pelletier/go-toml/parser.go b/vendor/github.com/pelletier/go-toml/parser.go index 20e90a3e..2d27599a 100644 --- a/vendor/github.com/pelletier/go-toml/parser.go +++ b/vendor/github.com/pelletier/go-toml/parser.go @@ -5,6 +5,7 @@ package toml import ( "errors" "fmt" + "math" "reflect" "regexp" "strconv" @@ -13,9 +14,9 @@ import ( ) type tomlParser struct { - flow chan token - tree *TomlTree - tokensBuffer []token + flowIdx int + flow []token + tree *Tree currentTable []string seenTableKeys []string } @@ -34,16 +35,10 @@ func (p *tomlParser) run() { } func (p *tomlParser) peek() *token { - if len(p.tokensBuffer) != 0 { - return &(p.tokensBuffer[0]) - } - - tok, ok := <-p.flow - if !ok { + if p.flowIdx >= len(p.flow) { return nil } - p.tokensBuffer = append(p.tokensBuffer, tok) - return &tok + return &p.flow[p.flowIdx] } func (p *tomlParser) assume(typ tokenType) { @@ -57,16 +52,12 @@ func (p *tomlParser) assume(typ tokenType) { } func (p *tomlParser) getToken() *token { - if len(p.tokensBuffer) != 0 { - tok := p.tokensBuffer[0] - p.tokensBuffer = p.tokensBuffer[1:] - return &tok - } - tok, ok := <-p.flow - if !ok { + tok := p.peek() + if tok == nil { return nil } - return &tok + p.flowIdx++ + return tok } func (p *tomlParser) parseStart() tomlParserStateFn { @@ -106,18 +97,18 @@ func (p *tomlParser) parseGroupArray() tomlParserStateFn { } p.tree.createSubTree(keys[:len(keys)-1], startToken.Position) // create parent entries destTree := p.tree.GetPath(keys) - var array []*TomlTree + var array []*Tree if destTree == nil { - array = make([]*TomlTree, 0) - } else if target, ok := destTree.([]*TomlTree); ok && target != nil { - array = destTree.([]*TomlTree) + array = make([]*Tree, 0) + } else if target, ok := destTree.([]*Tree); ok && target != nil { + array = destTree.([]*Tree) } else { p.raiseError(key, "key %s is already assigned and not of type table array", key) } p.currentTable = keys // add a new tree to the end of the table array - newTree := newTomlTree() + newTree := newTree() newTree.position = startToken.Position array = append(array, newTree) p.tree.SetPath(p.currentTable, array) @@ -183,11 +174,11 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { } // find the table to assign, looking out for arrays of tables - var targetNode *TomlTree + var targetNode *Tree switch node := p.tree.GetPath(tableKey).(type) { - case []*TomlTree: + case []*Tree: targetNode = node[len(node)-1] - case *TomlTree: + case *Tree: targetNode = node default: p.raiseError(key, "Unknown table type for path: %s", @@ -195,10 +186,7 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { } // assign value to the found table - keyVals, err := parseKey(key.val) - if err != nil { - p.raiseError(key, "%s", err) - } + keyVals := []string{key.val} if len(keyVals) != 1 { p.raiseError(key, "Invalid key") } @@ -212,23 +200,35 @@ func (p *tomlParser) parseAssign() tomlParserStateFn { var toInsert interface{} switch value.(type) { - case *TomlTree, []*TomlTree: + case *Tree, []*Tree: toInsert = value default: - toInsert = &tomlValue{value, key.Position} + toInsert = &tomlValue{value: value, position: key.Position} } targetNode.values[keyVal] = toInsert return p.parseStart } var numberUnderscoreInvalidRegexp *regexp.Regexp +var hexNumberUnderscoreInvalidRegexp *regexp.Regexp -func cleanupNumberToken(value string) (string, error) { +func numberContainsInvalidUnderscore(value string) error { if numberUnderscoreInvalidRegexp.MatchString(value) { - return "", errors.New("invalid use of _ in number") + return errors.New("invalid use of _ in number") } + return nil +} + +func hexNumberContainsInvalidUnderscore(value string) error { + if hexNumberUnderscoreInvalidRegexp.MatchString(value) { + return errors.New("invalid use of _ in hex number") + } + return nil +} + +func cleanupNumberToken(value string) string { cleanedVal := strings.Replace(value, "_", "", -1) - return cleanedVal, nil + return cleanedVal } func (p *tomlParser) parseRvalue() interface{} { @@ -244,21 +244,57 @@ func (p *tomlParser) parseRvalue() interface{} { return true case tokenFalse: return false - case tokenInteger: - cleanedVal, err := cleanupNumberToken(tok.val) - if err != nil { - p.raiseError(tok, "%s", err) + case tokenInf: + if tok.val[0] == '-' { + return math.Inf(-1) + } + return math.Inf(1) + case tokenNan: + return math.NaN() + case tokenInteger: + cleanedVal := cleanupNumberToken(tok.val) + var err error + var val int64 + if len(cleanedVal) >= 3 && cleanedVal[0] == '0' { + switch cleanedVal[1] { + case 'x': + err = hexNumberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err = strconv.ParseInt(cleanedVal[2:], 16, 64) + case 'o': + err = numberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err = strconv.ParseInt(cleanedVal[2:], 8, 64) + case 'b': + err = numberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err = strconv.ParseInt(cleanedVal[2:], 2, 64) + default: + panic("invalid base") // the lexer should catch this first + } + } else { + err = numberContainsInvalidUnderscore(tok.val) + if err != nil { + p.raiseError(tok, "%s", err) + } + val, err = strconv.ParseInt(cleanedVal, 10, 64) } - val, err := strconv.ParseInt(cleanedVal, 10, 64) if err != nil { p.raiseError(tok, "%s", err) } return val case tokenFloat: - cleanedVal, err := cleanupNumberToken(tok.val) + err := numberContainsInvalidUnderscore(tok.val) if err != nil { p.raiseError(tok, "%s", err) } + cleanedVal := cleanupNumberToken(tok.val) val, err := strconv.ParseFloat(cleanedVal, 64) if err != nil { p.raiseError(tok, "%s", err) @@ -289,8 +325,8 @@ func tokenIsComma(t *token) bool { return t != nil && t.typ == tokenComma } -func (p *tomlParser) parseInlineTable() *TomlTree { - tree := newTomlTree() +func (p *tomlParser) parseInlineTable() *Tree { + tree := newTree() var previous *token Loop: for { @@ -319,7 +355,7 @@ Loop: } p.getToken() default: - p.raiseError(follow, "unexpected token type in inline table: %s", follow.typ.String()) + p.raiseError(follow, "unexpected token type in inline table: %s", follow.String()) } previous = follow } @@ -360,27 +396,27 @@ func (p *tomlParser) parseArray() interface{} { p.getToken() } } - // An array of TomlTrees is actually an array of inline + // An array of Trees is actually an array of inline // tables, which is a shorthand for a table array. If the - // array was not converted from []interface{} to []*TomlTree, + // array was not converted from []interface{} to []*Tree, // the two notations would not be equivalent. - if arrayType == reflect.TypeOf(newTomlTree()) { - tomlArray := make([]*TomlTree, len(array)) + if arrayType == reflect.TypeOf(newTree()) { + tomlArray := make([]*Tree, len(array)) for i, v := range array { - tomlArray[i] = v.(*TomlTree) + tomlArray[i] = v.(*Tree) } return tomlArray } return array } -func parseToml(flow chan token) *TomlTree { - result := newTomlTree() +func parseToml(flow []token) *Tree { + result := newTree() result.position = Position{1, 1} parser := &tomlParser{ + flowIdx: 0, flow: flow, tree: result, - tokensBuffer: make([]token, 0), currentTable: make([]string, 0), seenTableKeys: make([]string, 0), } @@ -389,5 +425,6 @@ func parseToml(flow chan token) *TomlTree { } func init() { - numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d]|_$|^_)`) + numberUnderscoreInvalidRegexp = regexp.MustCompile(`([^\d]_|_[^\d])|_$|^_`) + hexNumberUnderscoreInvalidRegexp = regexp.MustCompile(`(^0x_)|([^\da-f]_|_[^\da-f])|_$|^_`) } diff --git a/vendor/github.com/pelletier/go-toml/query.go b/vendor/github.com/pelletier/go-toml/query.go deleted file mode 100644 index 307a1eca..00000000 --- a/vendor/github.com/pelletier/go-toml/query.go +++ /dev/null @@ -1,153 +0,0 @@ -package toml - -import ( - "time" -) - -// NodeFilterFn represents a user-defined filter function, for use with -// Query.SetFilter(). -// -// The return value of the function must indicate if 'node' is to be included -// at this stage of the TOML path. Returning true will include the node, and -// returning false will exclude it. -// -// NOTE: Care should be taken to write script callbacks such that they are safe -// to use from multiple goroutines. -type NodeFilterFn func(node interface{}) bool - -// QueryResult is the result of Executing a Query. -type QueryResult struct { - items []interface{} - positions []Position -} - -// appends a value/position pair to the result set. -func (r *QueryResult) appendResult(node interface{}, pos Position) { - r.items = append(r.items, node) - r.positions = append(r.positions, pos) -} - -// Values is a set of values within a QueryResult. The order of values is not -// guaranteed to be in document order, and may be different each time a query is -// executed. -func (r QueryResult) Values() []interface{} { - values := make([]interface{}, len(r.items)) - for i, v := range r.items { - o, ok := v.(*tomlValue) - if ok { - values[i] = o.value - } else { - values[i] = v - } - } - return values -} - -// Positions is a set of positions for values within a QueryResult. Each index -// in Positions() corresponds to the entry in Value() of the same index. -func (r QueryResult) Positions() []Position { - return r.positions -} - -// runtime context for executing query paths -type queryContext struct { - result *QueryResult - filters *map[string]NodeFilterFn - lastPosition Position -} - -// generic path functor interface -type pathFn interface { - setNext(next pathFn) - call(node interface{}, ctx *queryContext) -} - -// A Query is the representation of a compiled TOML path. A Query is safe -// for concurrent use by multiple goroutines. -type Query struct { - root pathFn - tail pathFn - filters *map[string]NodeFilterFn -} - -func newQuery() *Query { - return &Query{ - root: nil, - tail: nil, - filters: &defaultFilterFunctions, - } -} - -func (q *Query) appendPath(next pathFn) { - if q.root == nil { - q.root = next - } else { - q.tail.setNext(next) - } - q.tail = next - next.setNext(newTerminatingFn()) // init the next functor -} - -// CompileQuery compiles a TOML path expression. The returned Query can be used -// to match elements within a TomlTree and its descendants. -func CompileQuery(path string) (*Query, error) { - return parseQuery(lexQuery(path)) -} - -// Execute executes a query against a TomlTree, and returns the result of the query. -func (q *Query) Execute(tree *TomlTree) *QueryResult { - result := &QueryResult{ - items: []interface{}{}, - positions: []Position{}, - } - if q.root == nil { - result.appendResult(tree, tree.GetPosition("")) - } else { - ctx := &queryContext{ - result: result, - filters: q.filters, - } - q.root.call(tree, ctx) - } - return result -} - -// SetFilter sets a user-defined filter function. These may be used inside -// "?(..)" query expressions to filter TOML document elements within a query. -func (q *Query) SetFilter(name string, fn NodeFilterFn) { - if q.filters == &defaultFilterFunctions { - // clone the static table - q.filters = &map[string]NodeFilterFn{} - for k, v := range defaultFilterFunctions { - (*q.filters)[k] = v - } - } - (*q.filters)[name] = fn -} - -var defaultFilterFunctions = map[string]NodeFilterFn{ - "tree": func(node interface{}) bool { - _, ok := node.(*TomlTree) - return ok - }, - "int": func(node interface{}) bool { - _, ok := node.(int64) - return ok - }, - "float": func(node interface{}) bool { - _, ok := node.(float64) - return ok - }, - "string": func(node interface{}) bool { - _, ok := node.(string) - return ok - }, - "time": func(node interface{}) bool { - _, ok := node.(time.Time) - return ok - }, - "bool": func(node interface{}) bool { - _, ok := node.(bool) - return ok - }, -} diff --git a/vendor/github.com/pelletier/go-toml/querylexer.go b/vendor/github.com/pelletier/go-toml/querylexer.go deleted file mode 100644 index 960681d0..00000000 --- a/vendor/github.com/pelletier/go-toml/querylexer.go +++ /dev/null @@ -1,356 +0,0 @@ -// TOML JSONPath lexer. -// -// Written using the principles developed by Rob Pike in -// http://www.youtube.com/watch?v=HxaD_trXwRE - -package toml - -import ( - "fmt" - "strconv" - "strings" - "unicode/utf8" -) - -// Lexer state function -type queryLexStateFn func() queryLexStateFn - -// Lexer definition -type queryLexer struct { - input string - start int - pos int - width int - tokens chan token - depth int - line int - col int - stringTerm string -} - -func (l *queryLexer) run() { - for state := l.lexVoid; state != nil; { - state = state() - } - close(l.tokens) -} - -func (l *queryLexer) nextStart() { - // iterate by runes (utf8 characters) - // search for newlines and advance line/col counts - for i := l.start; i < l.pos; { - r, width := utf8.DecodeRuneInString(l.input[i:]) - if r == '\n' { - l.line++ - l.col = 1 - } else { - l.col++ - } - i += width - } - // advance start position to next token - l.start = l.pos -} - -func (l *queryLexer) emit(t tokenType) { - l.tokens <- token{ - Position: Position{l.line, l.col}, - typ: t, - val: l.input[l.start:l.pos], - } - l.nextStart() -} - -func (l *queryLexer) emitWithValue(t tokenType, value string) { - l.tokens <- token{ - Position: Position{l.line, l.col}, - typ: t, - val: value, - } - l.nextStart() -} - -func (l *queryLexer) next() rune { - if l.pos >= len(l.input) { - l.width = 0 - return eof - } - var r rune - r, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) - l.pos += l.width - return r -} - -func (l *queryLexer) ignore() { - l.nextStart() -} - -func (l *queryLexer) backup() { - l.pos -= l.width -} - -func (l *queryLexer) errorf(format string, args ...interface{}) queryLexStateFn { - l.tokens <- token{ - Position: Position{l.line, l.col}, - typ: tokenError, - val: fmt.Sprintf(format, args...), - } - return nil -} - -func (l *queryLexer) peek() rune { - r := l.next() - l.backup() - return r -} - -func (l *queryLexer) accept(valid string) bool { - if strings.ContainsRune(valid, l.next()) { - return true - } - l.backup() - return false -} - -func (l *queryLexer) follow(next string) bool { - return strings.HasPrefix(l.input[l.pos:], next) -} - -func (l *queryLexer) lexVoid() queryLexStateFn { - for { - next := l.peek() - switch next { - case '$': - l.pos++ - l.emit(tokenDollar) - continue - case '.': - if l.follow("..") { - l.pos += 2 - l.emit(tokenDotDot) - } else { - l.pos++ - l.emit(tokenDot) - } - continue - case '[': - l.pos++ - l.emit(tokenLeftBracket) - continue - case ']': - l.pos++ - l.emit(tokenRightBracket) - continue - case ',': - l.pos++ - l.emit(tokenComma) - continue - case '*': - l.pos++ - l.emit(tokenStar) - continue - case '(': - l.pos++ - l.emit(tokenLeftParen) - continue - case ')': - l.pos++ - l.emit(tokenRightParen) - continue - case '?': - l.pos++ - l.emit(tokenQuestion) - continue - case ':': - l.pos++ - l.emit(tokenColon) - continue - case '\'': - l.ignore() - l.stringTerm = string(next) - return l.lexString - case '"': - l.ignore() - l.stringTerm = string(next) - return l.lexString - } - - if isSpace(next) { - l.next() - l.ignore() - continue - } - - if isAlphanumeric(next) { - return l.lexKey - } - - if next == '+' || next == '-' || isDigit(next) { - return l.lexNumber - } - - if l.next() == eof { - break - } - - return l.errorf("unexpected char: '%v'", next) - } - l.emit(tokenEOF) - return nil -} - -func (l *queryLexer) lexKey() queryLexStateFn { - for { - next := l.peek() - if !isAlphanumeric(next) { - l.emit(tokenKey) - return l.lexVoid - } - - if l.next() == eof { - break - } - } - l.emit(tokenEOF) - return nil -} - -func (l *queryLexer) lexString() queryLexStateFn { - l.pos++ - l.ignore() - growingString := "" - - for { - if l.follow(l.stringTerm) { - l.emitWithValue(tokenString, growingString) - l.pos++ - l.ignore() - return l.lexVoid - } - - if l.follow("\\\"") { - l.pos++ - growingString += "\"" - } else if l.follow("\\'") { - l.pos++ - growingString += "'" - } else if l.follow("\\n") { - l.pos++ - growingString += "\n" - } else if l.follow("\\b") { - l.pos++ - growingString += "\b" - } else if l.follow("\\f") { - l.pos++ - growingString += "\f" - } else if l.follow("\\/") { - l.pos++ - growingString += "/" - } else if l.follow("\\t") { - l.pos++ - growingString += "\t" - } else if l.follow("\\r") { - l.pos++ - growingString += "\r" - } else if l.follow("\\\\") { - l.pos++ - growingString += "\\" - } else if l.follow("\\u") { - l.pos += 2 - code := "" - for i := 0; i < 4; i++ { - c := l.peek() - l.pos++ - if !isHexDigit(c) { - return l.errorf("unfinished unicode escape") - } - code = code + string(c) - } - l.pos-- - intcode, err := strconv.ParseInt(code, 16, 32) - if err != nil { - return l.errorf("invalid unicode escape: \\u" + code) - } - growingString += string(rune(intcode)) - } else if l.follow("\\U") { - l.pos += 2 - code := "" - for i := 0; i < 8; i++ { - c := l.peek() - l.pos++ - if !isHexDigit(c) { - return l.errorf("unfinished unicode escape") - } - code = code + string(c) - } - l.pos-- - intcode, err := strconv.ParseInt(code, 16, 32) - if err != nil { - return l.errorf("invalid unicode escape: \\u" + code) - } - growingString += string(rune(intcode)) - } else if l.follow("\\") { - l.pos++ - return l.errorf("invalid escape sequence: \\" + string(l.peek())) - } else { - growingString += string(l.peek()) - } - - if l.next() == eof { - break - } - } - - return l.errorf("unclosed string") -} - -func (l *queryLexer) lexNumber() queryLexStateFn { - l.ignore() - if !l.accept("+") { - l.accept("-") - } - pointSeen := false - digitSeen := false - for { - next := l.next() - if next == '.' { - if pointSeen { - return l.errorf("cannot have two dots in one float") - } - if !isDigit(l.peek()) { - return l.errorf("float cannot end with a dot") - } - pointSeen = true - } else if isDigit(next) { - digitSeen = true - } else { - l.backup() - break - } - if pointSeen && !digitSeen { - return l.errorf("cannot start float with a dot") - } - } - - if !digitSeen { - return l.errorf("no digit in that number") - } - if pointSeen { - l.emit(tokenFloat) - } else { - l.emit(tokenInteger) - } - return l.lexVoid -} - -// Entry point -func lexQuery(input string) chan token { - l := &queryLexer{ - input: input, - tokens: make(chan token), - line: 1, - col: 1, - } - go l.run() - return l.tokens -} diff --git a/vendor/github.com/pelletier/go-toml/queryparser.go b/vendor/github.com/pelletier/go-toml/queryparser.go deleted file mode 100644 index 1cbfc83b..00000000 --- a/vendor/github.com/pelletier/go-toml/queryparser.go +++ /dev/null @@ -1,275 +0,0 @@ -/* - Based on the "jsonpath" spec/concept. - - http://goessner.net/articles/JsonPath/ - https://code.google.com/p/json-path/ -*/ - -package toml - -import ( - "fmt" -) - -const maxInt = int(^uint(0) >> 1) - -type queryParser struct { - flow chan token - tokensBuffer []token - query *Query - union []pathFn - err error -} - -type queryParserStateFn func() queryParserStateFn - -// Formats and panics an error message based on a token -func (p *queryParser) parseError(tok *token, msg string, args ...interface{}) queryParserStateFn { - p.err = fmt.Errorf(tok.Position.String()+": "+msg, args...) - return nil // trigger parse to end -} - -func (p *queryParser) run() { - for state := p.parseStart; state != nil; { - state = state() - } -} - -func (p *queryParser) backup(tok *token) { - p.tokensBuffer = append(p.tokensBuffer, *tok) -} - -func (p *queryParser) peek() *token { - if len(p.tokensBuffer) != 0 { - return &(p.tokensBuffer[0]) - } - - tok, ok := <-p.flow - if !ok { - return nil - } - p.backup(&tok) - return &tok -} - -func (p *queryParser) lookahead(types ...tokenType) bool { - result := true - buffer := []token{} - - for _, typ := range types { - tok := p.getToken() - if tok == nil { - result = false - break - } - buffer = append(buffer, *tok) - if tok.typ != typ { - result = false - break - } - } - // add the tokens back to the buffer, and return - p.tokensBuffer = append(p.tokensBuffer, buffer...) - return result -} - -func (p *queryParser) getToken() *token { - if len(p.tokensBuffer) != 0 { - tok := p.tokensBuffer[0] - p.tokensBuffer = p.tokensBuffer[1:] - return &tok - } - tok, ok := <-p.flow - if !ok { - return nil - } - return &tok -} - -func (p *queryParser) parseStart() queryParserStateFn { - tok := p.getToken() - - if tok == nil || tok.typ == tokenEOF { - return nil - } - - if tok.typ != tokenDollar { - return p.parseError(tok, "Expected '$' at start of expression") - } - - return p.parseMatchExpr -} - -// handle '.' prefix, '[]', and '..' -func (p *queryParser) parseMatchExpr() queryParserStateFn { - tok := p.getToken() - switch tok.typ { - case tokenDotDot: - p.query.appendPath(&matchRecursiveFn{}) - // nested parse for '..' - tok := p.getToken() - switch tok.typ { - case tokenKey: - p.query.appendPath(newMatchKeyFn(tok.val)) - return p.parseMatchExpr - case tokenLeftBracket: - return p.parseBracketExpr - case tokenStar: - // do nothing - the recursive predicate is enough - return p.parseMatchExpr - } - - case tokenDot: - // nested parse for '.' - tok := p.getToken() - switch tok.typ { - case tokenKey: - p.query.appendPath(newMatchKeyFn(tok.val)) - return p.parseMatchExpr - case tokenStar: - p.query.appendPath(&matchAnyFn{}) - return p.parseMatchExpr - } - - case tokenLeftBracket: - return p.parseBracketExpr - - case tokenEOF: - return nil // allow EOF at this stage - } - return p.parseError(tok, "expected match expression") -} - -func (p *queryParser) parseBracketExpr() queryParserStateFn { - if p.lookahead(tokenInteger, tokenColon) { - return p.parseSliceExpr - } - if p.peek().typ == tokenColon { - return p.parseSliceExpr - } - return p.parseUnionExpr -} - -func (p *queryParser) parseUnionExpr() queryParserStateFn { - var tok *token - - // this state can be traversed after some sub-expressions - // so be careful when setting up state in the parser - if p.union == nil { - p.union = []pathFn{} - } - -loop: // labeled loop for easy breaking - for { - if len(p.union) > 0 { - // parse delimiter or terminator - tok = p.getToken() - switch tok.typ { - case tokenComma: - // do nothing - case tokenRightBracket: - break loop - default: - return p.parseError(tok, "expected ',' or ']', not '%s'", tok.val) - } - } - - // parse sub expression - tok = p.getToken() - switch tok.typ { - case tokenInteger: - p.union = append(p.union, newMatchIndexFn(tok.Int())) - case tokenKey: - p.union = append(p.union, newMatchKeyFn(tok.val)) - case tokenString: - p.union = append(p.union, newMatchKeyFn(tok.val)) - case tokenQuestion: - return p.parseFilterExpr - default: - return p.parseError(tok, "expected union sub expression, not '%s', %d", tok.val, len(p.union)) - } - } - - // if there is only one sub-expression, use that instead - if len(p.union) == 1 { - p.query.appendPath(p.union[0]) - } else { - p.query.appendPath(&matchUnionFn{p.union}) - } - - p.union = nil // clear out state - return p.parseMatchExpr -} - -func (p *queryParser) parseSliceExpr() queryParserStateFn { - // init slice to grab all elements - start, end, step := 0, maxInt, 1 - - // parse optional start - tok := p.getToken() - if tok.typ == tokenInteger { - start = tok.Int() - tok = p.getToken() - } - if tok.typ != tokenColon { - return p.parseError(tok, "expected ':'") - } - - // parse optional end - tok = p.getToken() - if tok.typ == tokenInteger { - end = tok.Int() - tok = p.getToken() - } - if tok.typ == tokenRightBracket { - p.query.appendPath(newMatchSliceFn(start, end, step)) - return p.parseMatchExpr - } - if tok.typ != tokenColon { - return p.parseError(tok, "expected ']' or ':'") - } - - // parse optional step - tok = p.getToken() - if tok.typ == tokenInteger { - step = tok.Int() - if step < 0 { - return p.parseError(tok, "step must be a positive value") - } - tok = p.getToken() - } - if tok.typ != tokenRightBracket { - return p.parseError(tok, "expected ']'") - } - - p.query.appendPath(newMatchSliceFn(start, end, step)) - return p.parseMatchExpr -} - -func (p *queryParser) parseFilterExpr() queryParserStateFn { - tok := p.getToken() - if tok.typ != tokenLeftParen { - return p.parseError(tok, "expected left-parenthesis for filter expression") - } - tok = p.getToken() - if tok.typ != tokenKey && tok.typ != tokenString { - return p.parseError(tok, "expected key or string for filter funciton name") - } - name := tok.val - tok = p.getToken() - if tok.typ != tokenRightParen { - return p.parseError(tok, "expected right-parenthesis for filter expression") - } - p.union = append(p.union, newMatchFilterFn(name, tok.Position)) - return p.parseUnionExpr -} - -func parseQuery(flow chan token) (*Query, error) { - parser := &queryParser{ - flow: flow, - tokensBuffer: []token{}, - query: newQuery(), - } - parser.run() - return parser.query, parser.err -} diff --git a/vendor/github.com/pelletier/go-toml/test.sh b/vendor/github.com/pelletier/go-toml/test.sh index 436d2fb6..ba6adf3f 100755 --- a/vendor/github.com/pelletier/go-toml/test.sh +++ b/vendor/github.com/pelletier/go-toml/test.sh @@ -1,6 +1,7 @@ #!/bin/bash # fail out of the script if anything here fails set -e +set -o pipefail # set the path to the present working directory export GOPATH=`pwd` @@ -19,11 +20,13 @@ function git_clone() { popd } -# Run go vet -go vet ./... +# Remove potential previous runs +rm -rf src test_program_bin toml-test go get github.com/pelletier/go-buffruneio go get github.com/davecgh/go-spew/spew +go get gopkg.in/yaml.v2 +go get github.com/BurntSushi/toml # get code for BurntSushi TOML validation # pinning all to 'HEAD' for version 0.3.x work (TODO: pin to commit hash when tests stabilize) @@ -36,13 +39,16 @@ go build -o toml-test github.com/BurntSushi/toml-test # vendorize the current lib for testing # NOTE: this basically mocks an install without having to go back out to github for code mkdir -p src/github.com/pelletier/go-toml/cmd +mkdir -p src/github.com/pelletier/go-toml/query cp *.go *.toml src/github.com/pelletier/go-toml cp -R cmd/* src/github.com/pelletier/go-toml/cmd +cp -R query/* src/github.com/pelletier/go-toml/query go build -o test_program_bin src/github.com/pelletier/go-toml/cmd/test_program.go # Run basic unit tests -go test github.com/pelletier/go-toml -v -covermode=count -coverprofile=coverage.out +go test github.com/pelletier/go-toml -covermode=count -coverprofile=coverage.out go test github.com/pelletier/go-toml/cmd/tomljson +go test github.com/pelletier/go-toml/query # run the entire BurntSushi test suite if [[ $# -eq 0 ]] ; then diff --git a/vendor/github.com/pelletier/go-toml/token.go b/vendor/github.com/pelletier/go-toml/token.go index 5581fe0b..1a908134 100644 --- a/vendor/github.com/pelletier/go-toml/token.go +++ b/vendor/github.com/pelletier/go-toml/token.go @@ -23,6 +23,8 @@ const ( tokenTrue tokenFalse tokenFloat + tokenInf + tokenNan tokenEqual tokenLeftBracket tokenRightBracket @@ -55,6 +57,8 @@ var tokenTypeNames = []string{ "True", "False", "Float", + "Inf", + "NaN", "=", "[", "]", diff --git a/vendor/github.com/pelletier/go-toml/toml.go b/vendor/github.com/pelletier/go-toml/toml.go index 1ba56a1c..05493a44 100644 --- a/vendor/github.com/pelletier/go-toml/toml.go +++ b/vendor/github.com/pelletier/go-toml/toml.go @@ -4,40 +4,50 @@ import ( "errors" "fmt" "io" + "io/ioutil" "os" "runtime" "strings" ) type tomlValue struct { - value interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list - position Position + value interface{} // string, int64, uint64, float64, bool, time.Time, [] of any of this list + comment string + commented bool + position Position } -// TomlTree is the result of the parsing of a TOML file. -type TomlTree struct { - values map[string]interface{} // string -> *tomlValue, *TomlTree, []*TomlTree - position Position +// Tree is the result of the parsing of a TOML file. +type Tree struct { + values map[string]interface{} // string -> *tomlValue, *Tree, []*Tree + comment string + commented bool + position Position } -func newTomlTree() *TomlTree { - return &TomlTree{ +func newTree() *Tree { + return &Tree{ values: make(map[string]interface{}), position: Position{}, } } -// TreeFromMap initializes a new TomlTree object using the given map. -func TreeFromMap(m map[string]interface{}) (*TomlTree, error) { +// TreeFromMap initializes a new Tree object using the given map. +func TreeFromMap(m map[string]interface{}) (*Tree, error) { result, err := toTree(m) if err != nil { return nil, err } - return result.(*TomlTree), nil + return result.(*Tree), nil +} + +// Position returns the position of the tree. +func (t *Tree) Position() Position { + return t.position } // Has returns a boolean indicating if the given key exists. -func (t *TomlTree) Has(key string) bool { +func (t *Tree) Has(key string) bool { if key == "" { return false } @@ -45,38 +55,36 @@ func (t *TomlTree) Has(key string) bool { } // HasPath returns true if the given path of keys exists, false otherwise. -func (t *TomlTree) HasPath(keys []string) bool { +func (t *Tree) HasPath(keys []string) bool { return t.GetPath(keys) != nil } -// Keys returns the keys of the toplevel tree. -// Warning: this is a costly operation. -func (t *TomlTree) Keys() []string { - var keys []string +// Keys returns the keys of the toplevel tree (does not recurse). +func (t *Tree) Keys() []string { + keys := make([]string, len(t.values)) + i := 0 for k := range t.values { - keys = append(keys, k) + keys[i] = k + i++ } return keys } -// Get the value at key in the TomlTree. -// Key is a dot-separated path (e.g. a.b.c). +// Get the value at key in the Tree. +// Key is a dot-separated path (e.g. a.b.c) without single/double quoted strings. +// If you need to retrieve non-bare keys, use GetPath. // Returns nil if the path does not exist in the tree. // If keys is of length zero, the current tree is returned. -func (t *TomlTree) Get(key string) interface{} { +func (t *Tree) Get(key string) interface{} { if key == "" { return t } - comps, err := parseKey(key) - if err != nil { - return nil - } - return t.GetPath(comps) + return t.GetPath(strings.Split(key, ".")) } // GetPath returns the element in the tree indicated by 'keys'. // If keys is of length zero, the current tree is returned. -func (t *TomlTree) GetPath(keys []string) interface{} { +func (t *Tree) GetPath(keys []string) interface{} { if len(keys) == 0 { return t } @@ -87,9 +95,9 @@ func (t *TomlTree) GetPath(keys []string) interface{} { return nil } switch node := value.(type) { - case *TomlTree: + case *Tree: subtree = node - case []*TomlTree: + case []*Tree: // go to most recent element if len(node) == 0 { return nil @@ -109,7 +117,7 @@ func (t *TomlTree) GetPath(keys []string) interface{} { } // GetPosition returns the position of the given key. -func (t *TomlTree) GetPosition(key string) Position { +func (t *Tree) GetPosition(key string) Position { if key == "" { return t.position } @@ -118,7 +126,7 @@ func (t *TomlTree) GetPosition(key string) Position { // GetPositionPath returns the element in the tree indicated by 'keys'. // If keys is of length zero, the current tree is returned. -func (t *TomlTree) GetPositionPath(keys []string) Position { +func (t *Tree) GetPositionPath(keys []string) Position { if len(keys) == 0 { return t.position } @@ -129,9 +137,9 @@ func (t *TomlTree) GetPositionPath(keys []string) Position { return Position{0, 0} } switch node := value.(type) { - case *TomlTree: + case *Tree: subtree = node - case []*TomlTree: + case []*Tree: // go to most recent element if len(node) == 0 { return Position{0, 0} @@ -145,9 +153,9 @@ func (t *TomlTree) GetPositionPath(keys []string) Position { switch node := subtree.values[keys[len(keys)-1]].(type) { case *tomlValue: return node.position - case *TomlTree: + case *Tree: return node.position - case []*TomlTree: + case []*Tree: // go to most recent element if len(node) == 0 { return Position{0, 0} @@ -159,7 +167,7 @@ func (t *TomlTree) GetPositionPath(keys []string) Position { } // GetDefault works like Get but with a default value -func (t *TomlTree) GetDefault(key string, def interface{}) interface{} { +func (t *Tree) GetDefault(key string, def interface{}) interface{} { val := t.Get(key) if val == nil { return def @@ -169,30 +177,42 @@ func (t *TomlTree) GetDefault(key string, def interface{}) interface{} { // Set an element in the tree. // Key is a dot-separated path (e.g. a.b.c). -// Creates all necessary intermediates trees, if needed. -func (t *TomlTree) Set(key string, value interface{}) { - t.SetPath(strings.Split(key, "."), value) +// Creates all necessary intermediate trees, if needed. +func (t *Tree) Set(key string, value interface{}) { + t.SetWithComment(key, "", false, value) +} + +// SetWithComment is the same as Set, but allows you to provide comment +// information to the key, that will be reused by Marshal(). +func (t *Tree) SetWithComment(key string, comment string, commented bool, value interface{}) { + t.SetPathWithComment(strings.Split(key, "."), comment, commented, value) } // SetPath sets an element in the tree. // Keys is an array of path elements (e.g. {"a","b","c"}). -// Creates all necessary intermediates trees, if needed. -func (t *TomlTree) SetPath(keys []string, value interface{}) { +// Creates all necessary intermediate trees, if needed. +func (t *Tree) SetPath(keys []string, value interface{}) { + t.SetPathWithComment(keys, "", false, value) +} + +// SetPathWithComment is the same as SetPath, but allows you to provide comment +// information to the key, that will be reused by Marshal(). +func (t *Tree) SetPathWithComment(keys []string, comment string, commented bool, value interface{}) { subtree := t for _, intermediateKey := range keys[:len(keys)-1] { nextTree, exists := subtree.values[intermediateKey] if !exists { - nextTree = newTomlTree() + nextTree = newTree() subtree.values[intermediateKey] = nextTree // add new element here } switch node := nextTree.(type) { - case *TomlTree: + case *Tree: subtree = node - case []*TomlTree: + case []*Tree: // go to most recent element if len(node) == 0 { // create element if it does not exist - subtree.values[intermediateKey] = append(node, newTomlTree()) + subtree.values[intermediateKey] = append(node, newTree()) } subtree = node[len(node)-1] } @@ -201,14 +221,18 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) { var toInsert interface{} switch value.(type) { - case *TomlTree: + case *Tree: + tt := value.(*Tree) + tt.comment = comment toInsert = value - case []*TomlTree: + case []*Tree: toInsert = value case *tomlValue: - toInsert = value + tt := value.(*tomlValue) + tt.comment = comment + toInsert = tt default: - toInsert = &tomlValue{value: value} + toInsert = &tomlValue{value: value, comment: comment, commented: commented} } subtree.values[keys[len(keys)-1]] = toInsert @@ -221,21 +245,21 @@ func (t *TomlTree) SetPath(keys []string, value interface{}) { // and tree[a][b][c] // // Returns nil on success, error object on failure -func (t *TomlTree) createSubTree(keys []string, pos Position) error { +func (t *Tree) createSubTree(keys []string, pos Position) error { subtree := t for _, intermediateKey := range keys { nextTree, exists := subtree.values[intermediateKey] if !exists { - tree := newTomlTree() + tree := newTree() tree.position = pos subtree.values[intermediateKey] = tree nextTree = tree } switch node := nextTree.(type) { - case []*TomlTree: + case []*Tree: subtree = node[len(node)-1] - case *TomlTree: + case *Tree: subtree = node default: return fmt.Errorf("unknown type for path %s (%s): %T (%#v)", @@ -245,17 +269,8 @@ func (t *TomlTree) createSubTree(keys []string, pos Position) error { return nil } -// Query compiles and executes a query on a tree and returns the query result. -func (t *TomlTree) Query(query string) (*QueryResult, error) { - q, err := CompileQuery(query) - if err != nil { - return nil, err - } - return q.Execute(t), nil -} - -// LoadReader creates a TomlTree from any io.Reader. -func LoadReader(reader io.Reader) (tree *TomlTree, err error) { +// LoadBytes creates a Tree from a []byte. +func LoadBytes(b []byte) (tree *Tree, err error) { defer func() { if r := recover(); r != nil { if _, ok := r.(runtime.Error); ok { @@ -264,17 +279,27 @@ func LoadReader(reader io.Reader) (tree *TomlTree, err error) { err = errors.New(r.(string)) } }() - tree = parseToml(lexToml(reader)) + tree = parseToml(lexToml(b)) return } -// Load creates a TomlTree from a string. -func Load(content string) (tree *TomlTree, err error) { - return LoadReader(strings.NewReader(content)) +// LoadReader creates a Tree from any io.Reader. +func LoadReader(reader io.Reader) (tree *Tree, err error) { + inputBytes, err := ioutil.ReadAll(reader) + if err != nil { + return + } + tree, err = LoadBytes(inputBytes) + return } -// LoadFile creates a TomlTree from a file. -func LoadFile(path string) (tree *TomlTree, err error) { +// Load creates a Tree from a string. +func Load(content string) (tree *Tree, err error) { + return LoadBytes([]byte(content)) +} + +// LoadFile creates a Tree from a file. +func LoadFile(path string) (tree *Tree, err error) { file, err := os.Open(path) if err != nil { return nil, err diff --git a/vendor/github.com/pelletier/go-toml/tomltree_create.go b/vendor/github.com/pelletier/go-toml/tomltree_create.go index c6054f35..79610e9b 100644 --- a/vendor/github.com/pelletier/go-toml/tomltree_create.go +++ b/vendor/github.com/pelletier/go-toml/tomltree_create.go @@ -6,10 +6,7 @@ import ( "time" ) -// supported values: -// string, bool, int64, uint64, float64, time.Time, int, int8, int16, int32, uint, uint8, uint16, uint32, float32 - -var kindToTypeMapping = map[reflect.Kind]reflect.Type{ +var kindToType = [reflect.String + 1]reflect.Type{ reflect.Bool: reflect.TypeOf(true), reflect.String: reflect.TypeOf(""), reflect.Float32: reflect.TypeOf(float64(1)), @@ -26,6 +23,16 @@ var kindToTypeMapping = map[reflect.Kind]reflect.Type{ reflect.Uint64: reflect.TypeOf(uint64(1)), } +// typeFor returns a reflect.Type for a reflect.Kind, or nil if none is found. +// supported values: +// string, bool, int64, uint64, float64, time.Time, int, int8, int16, int32, uint, uint8, uint16, uint32, float32 +func typeFor(k reflect.Kind) reflect.Type { + if k > 0 && int(k) < len(kindToType) { + return kindToType[k] + } + return nil +} + func simpleValueCoercion(object interface{}) (interface{}, error) { switch original := object.(type) { case string, bool, int64, uint64, float64, time.Time: @@ -51,7 +58,7 @@ func simpleValueCoercion(object interface{}) (interface{}, error) { case fmt.Stringer: return original.String(), nil default: - return nil, fmt.Errorf("cannot convert type %T to TomlTree", object) + return nil, fmt.Errorf("cannot convert type %T to Tree", object) } } @@ -59,7 +66,7 @@ func sliceToTree(object interface{}) (interface{}, error) { // arrays are a bit tricky, since they can represent either a // collection of simple values, which is represented by one // *tomlValue, or an array of tables, which is represented by an - // array of *TomlTree. + // array of *Tree. // holding the assumption that this function is called from toTree only when value.Kind() is Array or Slice value := reflect.ValueOf(object) @@ -70,19 +77,19 @@ func sliceToTree(object interface{}) (interface{}, error) { } if insideType.Kind() == reflect.Map { // this is considered as an array of tables - tablesArray := make([]*TomlTree, 0, length) + tablesArray := make([]*Tree, 0, length) for i := 0; i < length; i++ { table := value.Index(i) tree, err := toTree(table.Interface()) if err != nil { return nil, err } - tablesArray = append(tablesArray, tree.(*TomlTree)) + tablesArray = append(tablesArray, tree.(*Tree)) } return tablesArray, nil } - sliceType := kindToTypeMapping[insideType.Kind()] + sliceType := typeFor(insideType.Kind()) if sliceType == nil { sliceType = insideType } @@ -97,7 +104,7 @@ func sliceToTree(object interface{}) (interface{}, error) { } arrayValue = reflect.Append(arrayValue, reflect.ValueOf(simpleValue)) } - return &tomlValue{arrayValue.Interface(), Position{}}, nil + return &tomlValue{value: arrayValue.Interface(), position: Position{}}, nil } func toTree(object interface{}) (interface{}, error) { @@ -120,7 +127,7 @@ func toTree(object interface{}) (interface{}, error) { } values[key.String()] = newValue } - return &TomlTree{values, Position{}}, nil + return &Tree{values: values, position: Position{}}, nil } if value.Kind() == reflect.Array || value.Kind() == reflect.Slice { @@ -131,5 +138,5 @@ func toTree(object interface{}) (interface{}, error) { if err != nil { return nil, err } - return &tomlValue{simpleValue, Position{}}, nil + return &tomlValue{value: simpleValue, position: Position{}}, nil } diff --git a/vendor/github.com/pelletier/go-toml/tomltree_write.go b/vendor/github.com/pelletier/go-toml/tomltree_write.go index 6a7fa174..b5600a58 100644 --- a/vendor/github.com/pelletier/go-toml/tomltree_write.go +++ b/vendor/github.com/pelletier/go-toml/tomltree_write.go @@ -4,57 +4,64 @@ import ( "bytes" "fmt" "io" + "math" + "reflect" "sort" "strconv" "strings" "time" - "reflect" ) // encodes a string to a TOML-compliant string value func encodeTomlString(value string) string { - result := "" + var b bytes.Buffer + for _, rr := range value { switch rr { case '\b': - result += "\\b" + b.WriteString(`\b`) case '\t': - result += "\\t" + b.WriteString(`\t`) case '\n': - result += "\\n" + b.WriteString(`\n`) case '\f': - result += "\\f" + b.WriteString(`\f`) case '\r': - result += "\\r" + b.WriteString(`\r`) case '"': - result += "\\\"" + b.WriteString(`\"`) case '\\': - result += "\\\\" + b.WriteString(`\\`) default: intRr := uint16(rr) if intRr < 0x001F { - result += fmt.Sprintf("\\u%0.4X", intRr) + b.WriteString(fmt.Sprintf("\\u%0.4X", intRr)) } else { - result += string(rr) + b.WriteRune(rr) } } } - return result + return b.String() } -func tomlValueStringRepresentation(v interface{}) (string, error) { +func tomlValueStringRepresentation(v interface{}, indent string, arraysOneElementPerLine bool) (string, error) { switch value := v.(type) { case uint64: return strconv.FormatUint(value, 10), nil case int64: return strconv.FormatInt(value, 10), nil case float64: - return strconv.FormatFloat(value, 'f', -1, 32), nil + // Ensure a round float does contain a decimal point. Otherwise feeding + // the output back to the parser would convert to an integer. + if math.Trunc(value) == value { + return strings.ToLower(strconv.FormatFloat(value, 'f', 1, 32)), nil + } + return strings.ToLower(strconv.FormatFloat(value, 'f', -1, 32)), nil case string: return "\"" + encodeTomlString(value) + "\"", nil case []byte: b, _ := v.([]byte) - return tomlValueStringRepresentation(string(b)) + return tomlValueStringRepresentation(string(b), indent, arraysOneElementPerLine) case bool: if value { return "true", nil @@ -69,28 +76,45 @@ func tomlValueStringRepresentation(v interface{}) (string, error) { rv := reflect.ValueOf(v) if rv.Kind() == reflect.Slice { - values := []string{} + var values []string for i := 0; i < rv.Len(); i++ { item := rv.Index(i).Interface() - itemRepr, err := tomlValueStringRepresentation(item) + itemRepr, err := tomlValueStringRepresentation(item, indent, arraysOneElementPerLine) if err != nil { return "", err } values = append(values, itemRepr) } + if arraysOneElementPerLine && len(values) > 1 { + stringBuffer := bytes.Buffer{} + valueIndent := indent + ` ` // TODO: move that to a shared encoder state + + stringBuffer.WriteString("[\n") + + for _, value := range values { + stringBuffer.WriteString(valueIndent) + stringBuffer.WriteString(value) + stringBuffer.WriteString(`,`) + stringBuffer.WriteString("\n") + } + + stringBuffer.WriteString(indent + "]") + + return stringBuffer.String(), nil + } return "[" + strings.Join(values, ",") + "]", nil } return "", fmt.Errorf("unsupported value type %T: %v", v, v) } -func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64) (int64, error) { +func (t *Tree) writeTo(w io.Writer, indent, keyspace string, bytesCount int64, arraysOneElementPerLine bool) (int64, error) { simpleValuesKeys := make([]string, 0) complexValuesKeys := make([]string, 0) for k := range t.values { v := t.values[k] switch v.(type) { - case *TomlTree, []*TomlTree: + case *Tree, []*Tree: complexValuesKeys = append(complexValuesKeys, k) default: simpleValuesKeys = append(simpleValuesKeys, k) @@ -106,13 +130,29 @@ func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int6 return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k]) } - repr, err := tomlValueStringRepresentation(v.value) + repr, err := tomlValueStringRepresentation(v.value, indent, arraysOneElementPerLine) if err != nil { return bytesCount, err } - kvRepr := fmt.Sprintf("%s%s = %s\n", indent, k, repr) - writtenBytesCount, err := w.Write([]byte(kvRepr)) + if v.comment != "" { + comment := strings.Replace(v.comment, "\n", "\n"+indent+"#", -1) + start := "# " + if strings.HasPrefix(comment, "#") { + start = "" + } + writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment, "\n") + bytesCount += int64(writtenBytesCountComment) + if errc != nil { + return bytesCount, errc + } + } + + var commented string + if v.commented { + commented = "# " + } + writtenBytesCount, err := writeStrings(w, indent, commented, k, " = ", repr, "\n") bytesCount += int64(writtenBytesCount) if err != nil { return bytesCount, err @@ -126,34 +166,50 @@ func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int6 if keyspace != "" { combinedKey = keyspace + "." + combinedKey } + var commented string + if t.commented { + commented = "# " + } switch node := v.(type) { // node has to be of those two types given how keys are sorted above - case *TomlTree: - tableName := fmt.Sprintf("\n%s[%s]\n", indent, combinedKey) - writtenBytesCount, err := w.Write([]byte(tableName)) + case *Tree: + tv, ok := t.values[k].(*Tree) + if !ok { + return bytesCount, fmt.Errorf("invalid value type at %s: %T", k, t.values[k]) + } + if tv.comment != "" { + comment := strings.Replace(tv.comment, "\n", "\n"+indent+"#", -1) + start := "# " + if strings.HasPrefix(comment, "#") { + start = "" + } + writtenBytesCountComment, errc := writeStrings(w, "\n", indent, start, comment) + bytesCount += int64(writtenBytesCountComment) + if errc != nil { + return bytesCount, errc + } + } + writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[", combinedKey, "]\n") bytesCount += int64(writtenBytesCount) if err != nil { return bytesCount, err } - bytesCount, err = node.writeTo(w, indent+" ", combinedKey, bytesCount) + bytesCount, err = node.writeTo(w, indent+" ", combinedKey, bytesCount, arraysOneElementPerLine) if err != nil { return bytesCount, err } - case []*TomlTree: + case []*Tree: for _, subTree := range node { - if len(subTree.values) > 0 { - tableArrayName := fmt.Sprintf("\n%s[[%s]]\n", indent, combinedKey) - writtenBytesCount, err := w.Write([]byte(tableArrayName)) - bytesCount += int64(writtenBytesCount) - if err != nil { - return bytesCount, err - } + writtenBytesCount, err := writeStrings(w, "\n", indent, commented, "[[", combinedKey, "]]\n") + bytesCount += int64(writtenBytesCount) + if err != nil { + return bytesCount, err + } - bytesCount, err = subTree.writeTo(w, indent+" ", combinedKey, bytesCount) - if err != nil { - return bytesCount, err - } + bytesCount, err = subTree.writeTo(w, indent+" ", combinedKey, bytesCount, arraysOneElementPerLine) + if err != nil { + return bytesCount, err } } } @@ -162,16 +218,28 @@ func (t *TomlTree) writeTo(w io.Writer, indent, keyspace string, bytesCount int6 return bytesCount, nil } -// WriteTo encode the TomlTree as Toml and writes it to the writer w. +func writeStrings(w io.Writer, s ...string) (int, error) { + var n int + for i := range s { + b, err := io.WriteString(w, s[i]) + n += b + if err != nil { + return n, err + } + } + return n, nil +} + +// WriteTo encode the Tree as Toml and writes it to the writer w. // Returns the number of bytes written in case of success, or an error if anything happened. -func (t *TomlTree) WriteTo(w io.Writer) (int64, error) { - return t.writeTo(w, "", "", 0) +func (t *Tree) WriteTo(w io.Writer) (int64, error) { + return t.writeTo(w, "", "", 0, false) } // ToTomlString generates a human-readable representation of the current tree. // Output spans multiple lines, and is suitable for ingest by a TOML parser. // If the conversion cannot be performed, ToString returns a non-nil error. -func (t *TomlTree) ToTomlString() (string, error) { +func (t *Tree) ToTomlString() (string, error) { var buf bytes.Buffer _, err := t.WriteTo(&buf) if err != nil { @@ -182,32 +250,34 @@ func (t *TomlTree) ToTomlString() (string, error) { // String generates a human-readable representation of the current tree. // Alias of ToString. Present to implement the fmt.Stringer interface. -func (t *TomlTree) String() string { +func (t *Tree) String() string { result, _ := t.ToTomlString() return result } // ToMap recursively generates a representation of the tree using Go built-in structures. // The following types are used: -// * uint64 -// * int64 -// * bool -// * string -// * time.Time -// * map[string]interface{} (where interface{} is any of this list) -// * []interface{} (where interface{} is any of this list) -func (t *TomlTree) ToMap() map[string]interface{} { +// +// * bool +// * float64 +// * int64 +// * string +// * uint64 +// * time.Time +// * map[string]interface{} (where interface{} is any of this list) +// * []interface{} (where interface{} is any of this list) +func (t *Tree) ToMap() map[string]interface{} { result := map[string]interface{}{} for k, v := range t.values { switch node := v.(type) { - case []*TomlTree: + case []*Tree: var array []interface{} for _, item := range node { array = append(array, item.ToMap()) } result[k] = array - case *TomlTree: + case *Tree: result[k] = node.ToMap() case *tomlValue: result[k] = node.value diff --git a/vendor/github.com/philhofer/fwd/LICENSE.md b/vendor/github.com/philhofer/fwd/LICENSE.md new file mode 100644 index 00000000..1ac6a81f --- /dev/null +++ b/vendor/github.com/philhofer/fwd/LICENSE.md @@ -0,0 +1,7 @@ +Copyright (c) 2014-2015, Philip Hofer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/philhofer/fwd/README.md b/vendor/github.com/philhofer/fwd/README.md new file mode 100644 index 00000000..38349af3 --- /dev/null +++ b/vendor/github.com/philhofer/fwd/README.md @@ -0,0 +1,315 @@ + +# fwd + import "github.com/philhofer/fwd" + +The `fwd` package provides a buffered reader +and writer. Each has methods that help improve +the encoding/decoding performance of some binary +protocols. + +The `fwd.Writer` and `fwd.Reader` type provide similar +functionality to their counterparts in `bufio`, plus +a few extra utility methods that simplify read-ahead +and write-ahead. I wrote this package to improve serialization +performance for http://github.com/tinylib/msgp, +where it provided about a 2x speedup over `bufio` for certain +workloads. However, care must be taken to understand the semantics of the +extra methods provided by this package, as they allow +the user to access and manipulate the buffer memory +directly. + +The extra methods for `fwd.Reader` are `Peek`, `Skip` +and `Next`. `(*fwd.Reader).Peek`, unlike `(*bufio.Reader).Peek`, +will re-allocate the read buffer in order to accommodate arbitrarily +large read-ahead. `(*fwd.Reader).Skip` skips the next `n` bytes +in the stream, and uses the `io.Seeker` interface if the underlying +stream implements it. `(*fwd.Reader).Next` returns a slice pointing +to the next `n` bytes in the read buffer (like `Peek`), but also +increments the read position. This allows users to process streams +in arbitrary block sizes without having to manage appropriately-sized +slices. Additionally, obviating the need to copy the data from the +buffer to another location in memory can improve performance dramatically +in CPU-bound applications. + +`fwd.Writer` only has one extra method, which is `(*fwd.Writer).Next`, which +returns a slice pointing to the next `n` bytes of the writer, and increments +the write position by the length of the returned slice. This allows users +to write directly to the end of the buffer. + + + + +## Constants +``` go +const ( + // DefaultReaderSize is the default size of the read buffer + DefaultReaderSize = 2048 +) +``` +``` go +const ( + // DefaultWriterSize is the + // default write buffer size. + DefaultWriterSize = 2048 +) +``` + + + +## type Reader +``` go +type Reader struct { + // contains filtered or unexported fields +} +``` +Reader is a buffered look-ahead reader + + + + + + + + + +### func NewReader +``` go +func NewReader(r io.Reader) *Reader +``` +NewReader returns a new *Reader that reads from 'r' + + +### func NewReaderSize +``` go +func NewReaderSize(r io.Reader, n int) *Reader +``` +NewReaderSize returns a new *Reader that +reads from 'r' and has a buffer size 'n' + + + + +### func (\*Reader) BufferSize +``` go +func (r *Reader) BufferSize() int +``` +BufferSize returns the total size of the buffer + + + +### func (\*Reader) Buffered +``` go +func (r *Reader) Buffered() int +``` +Buffered returns the number of bytes currently in the buffer + + + +### func (\*Reader) Next +``` go +func (r *Reader) Next(n int) ([]byte, error) +``` +Next returns the next 'n' bytes in the stream. +Unlike Peek, Next advances the reader position. +The returned bytes point to the same +data as the buffer, so the slice is +only valid until the next reader method call. +An EOF is considered an unexpected error. +If an the returned slice is less than the +length asked for, an error will be returned, +and the reader position will not be incremented. + + + +### func (\*Reader) Peek +``` go +func (r *Reader) Peek(n int) ([]byte, error) +``` +Peek returns the next 'n' buffered bytes, +reading from the underlying reader if necessary. +It will only return a slice shorter than 'n' bytes +if it also returns an error. Peek does not advance +the reader. EOF errors are *not* returned as +io.ErrUnexpectedEOF. + + + +### func (\*Reader) Read +``` go +func (r *Reader) Read(b []byte) (int, error) +``` +Read implements `io.Reader` + + + +### func (\*Reader) ReadByte +``` go +func (r *Reader) ReadByte() (byte, error) +``` +ReadByte implements `io.ByteReader` + + + +### func (\*Reader) ReadFull +``` go +func (r *Reader) ReadFull(b []byte) (int, error) +``` +ReadFull attempts to read len(b) bytes into +'b'. It returns the number of bytes read into +'b', and an error if it does not return len(b). +EOF is considered an unexpected error. + + + +### func (\*Reader) Reset +``` go +func (r *Reader) Reset(rd io.Reader) +``` +Reset resets the underlying reader +and the read buffer. + + + +### func (\*Reader) Skip +``` go +func (r *Reader) Skip(n int) (int, error) +``` +Skip moves the reader forward 'n' bytes. +Returns the number of bytes skipped and any +errors encountered. It is analogous to Seek(n, 1). +If the underlying reader implements io.Seeker, then +that method will be used to skip forward. + +If the reader encounters +an EOF before skipping 'n' bytes, it +returns io.ErrUnexpectedEOF. If the +underlying reader implements io.Seeker, then +those rules apply instead. (Many implementations +will not return `io.EOF` until the next call +to Read.) + + + +### func (\*Reader) WriteTo +``` go +func (r *Reader) WriteTo(w io.Writer) (int64, error) +``` +WriteTo implements `io.WriterTo` + + + +## type Writer +``` go +type Writer struct { + // contains filtered or unexported fields +} +``` +Writer is a buffered writer + + + + + + + + + +### func NewWriter +``` go +func NewWriter(w io.Writer) *Writer +``` +NewWriter returns a new writer +that writes to 'w' and has a buffer +that is `DefaultWriterSize` bytes. + + +### func NewWriterSize +``` go +func NewWriterSize(w io.Writer, size int) *Writer +``` +NewWriterSize returns a new writer +that writes to 'w' and has a buffer +that is 'size' bytes. + + + + +### func (\*Writer) BufferSize +``` go +func (w *Writer) BufferSize() int +``` +BufferSize returns the maximum size of the buffer. + + + +### func (\*Writer) Buffered +``` go +func (w *Writer) Buffered() int +``` +Buffered returns the number of buffered bytes +in the reader. + + + +### func (\*Writer) Flush +``` go +func (w *Writer) Flush() error +``` +Flush flushes any buffered bytes +to the underlying writer. + + + +### func (\*Writer) Next +``` go +func (w *Writer) Next(n int) ([]byte, error) +``` +Next returns the next 'n' free bytes +in the write buffer, flushing the writer +as necessary. Next will return `io.ErrShortBuffer` +if 'n' is greater than the size of the write buffer. +Calls to 'next' increment the write position by +the size of the returned buffer. + + + +### func (\*Writer) ReadFrom +``` go +func (w *Writer) ReadFrom(r io.Reader) (int64, error) +``` +ReadFrom implements `io.ReaderFrom` + + + +### func (\*Writer) Write +``` go +func (w *Writer) Write(p []byte) (int, error) +``` +Write implements `io.Writer` + + + +### func (\*Writer) WriteByte +``` go +func (w *Writer) WriteByte(b byte) error +``` +WriteByte implements `io.ByteWriter` + + + +### func (\*Writer) WriteString +``` go +func (w *Writer) WriteString(s string) (int, error) +``` +WriteString is analogous to Write, but it takes a string. + + + + + + + + + +- - - +Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md) \ No newline at end of file diff --git a/vendor/github.com/philhofer/fwd/reader.go b/vendor/github.com/philhofer/fwd/reader.go new file mode 100644 index 00000000..75be62ab --- /dev/null +++ b/vendor/github.com/philhofer/fwd/reader.go @@ -0,0 +1,383 @@ +// The `fwd` package provides a buffered reader +// and writer. Each has methods that help improve +// the encoding/decoding performance of some binary +// protocols. +// +// The `fwd.Writer` and `fwd.Reader` type provide similar +// functionality to their counterparts in `bufio`, plus +// a few extra utility methods that simplify read-ahead +// and write-ahead. I wrote this package to improve serialization +// performance for http://github.com/tinylib/msgp, +// where it provided about a 2x speedup over `bufio` for certain +// workloads. However, care must be taken to understand the semantics of the +// extra methods provided by this package, as they allow +// the user to access and manipulate the buffer memory +// directly. +// +// The extra methods for `fwd.Reader` are `Peek`, `Skip` +// and `Next`. `(*fwd.Reader).Peek`, unlike `(*bufio.Reader).Peek`, +// will re-allocate the read buffer in order to accommodate arbitrarily +// large read-ahead. `(*fwd.Reader).Skip` skips the next `n` bytes +// in the stream, and uses the `io.Seeker` interface if the underlying +// stream implements it. `(*fwd.Reader).Next` returns a slice pointing +// to the next `n` bytes in the read buffer (like `Peek`), but also +// increments the read position. This allows users to process streams +// in arbitrary block sizes without having to manage appropriately-sized +// slices. Additionally, obviating the need to copy the data from the +// buffer to another location in memory can improve performance dramatically +// in CPU-bound applications. +// +// `fwd.Writer` only has one extra method, which is `(*fwd.Writer).Next`, which +// returns a slice pointing to the next `n` bytes of the writer, and increments +// the write position by the length of the returned slice. This allows users +// to write directly to the end of the buffer. +// +package fwd + +import "io" + +const ( + // DefaultReaderSize is the default size of the read buffer + DefaultReaderSize = 2048 + + // minimum read buffer; straight from bufio + minReaderSize = 16 +) + +// NewReader returns a new *Reader that reads from 'r' +func NewReader(r io.Reader) *Reader { + return NewReaderSize(r, DefaultReaderSize) +} + +// NewReaderSize returns a new *Reader that +// reads from 'r' and has a buffer size 'n' +func NewReaderSize(r io.Reader, n int) *Reader { + rd := &Reader{ + r: r, + data: make([]byte, 0, max(minReaderSize, n)), + } + if s, ok := r.(io.Seeker); ok { + rd.rs = s + } + return rd +} + +// Reader is a buffered look-ahead reader +type Reader struct { + r io.Reader // underlying reader + + // data[n:len(data)] is buffered data; data[len(data):cap(data)] is free buffer space + data []byte // data + n int // read offset + state error // last read error + + // if the reader past to NewReader was + // also an io.Seeker, this is non-nil + rs io.Seeker +} + +// Reset resets the underlying reader +// and the read buffer. +func (r *Reader) Reset(rd io.Reader) { + r.r = rd + r.data = r.data[0:0] + r.n = 0 + r.state = nil + if s, ok := rd.(io.Seeker); ok { + r.rs = s + } else { + r.rs = nil + } +} + +// more() does one read on the underlying reader +func (r *Reader) more() { + // move data backwards so that + // the read offset is 0; this way + // we can supply the maximum number of + // bytes to the reader + if r.n != 0 { + if r.n < len(r.data) { + r.data = r.data[:copy(r.data[0:], r.data[r.n:])] + } else { + r.data = r.data[:0] + } + r.n = 0 + } + var a int + a, r.state = r.r.Read(r.data[len(r.data):cap(r.data)]) + if a == 0 && r.state == nil { + r.state = io.ErrNoProgress + return + } else if a > 0 && r.state == io.EOF { + // discard the io.EOF if we read more than 0 bytes. + // the next call to Read should return io.EOF again. + r.state = nil + } + r.data = r.data[:len(r.data)+a] +} + +// pop error +func (r *Reader) err() (e error) { + e, r.state = r.state, nil + return +} + +// pop error; EOF -> io.ErrUnexpectedEOF +func (r *Reader) noEOF() (e error) { + e, r.state = r.state, nil + if e == io.EOF { + e = io.ErrUnexpectedEOF + } + return +} + +// buffered bytes +func (r *Reader) buffered() int { return len(r.data) - r.n } + +// Buffered returns the number of bytes currently in the buffer +func (r *Reader) Buffered() int { return len(r.data) - r.n } + +// BufferSize returns the total size of the buffer +func (r *Reader) BufferSize() int { return cap(r.data) } + +// Peek returns the next 'n' buffered bytes, +// reading from the underlying reader if necessary. +// It will only return a slice shorter than 'n' bytes +// if it also returns an error. Peek does not advance +// the reader. EOF errors are *not* returned as +// io.ErrUnexpectedEOF. +func (r *Reader) Peek(n int) ([]byte, error) { + // in the degenerate case, + // we may need to realloc + // (the caller asked for more + // bytes than the size of the buffer) + if cap(r.data) < n { + old := r.data[r.n:] + r.data = make([]byte, n+r.buffered()) + r.data = r.data[:copy(r.data, old)] + r.n = 0 + } + + // keep filling until + // we hit an error or + // read enough bytes + for r.buffered() < n && r.state == nil { + r.more() + } + + // we must have hit an error + if r.buffered() < n { + return r.data[r.n:], r.err() + } + + return r.data[r.n : r.n+n], nil +} + +// Skip moves the reader forward 'n' bytes. +// Returns the number of bytes skipped and any +// errors encountered. It is analogous to Seek(n, 1). +// If the underlying reader implements io.Seeker, then +// that method will be used to skip forward. +// +// If the reader encounters +// an EOF before skipping 'n' bytes, it +// returns io.ErrUnexpectedEOF. If the +// underlying reader implements io.Seeker, then +// those rules apply instead. (Many implementations +// will not return `io.EOF` until the next call +// to Read.) +func (r *Reader) Skip(n int) (int, error) { + + // fast path + if r.buffered() >= n { + r.n += n + return n, nil + } + + // use seeker implementation + // if we can + if r.rs != nil { + return r.skipSeek(n) + } + + // loop on filling + // and then erasing + o := n + for r.buffered() < n && r.state == nil { + r.more() + // we can skip forward + // up to r.buffered() bytes + step := min(r.buffered(), n) + r.n += step + n -= step + } + // at this point, n should be + // 0 if everything went smoothly + return o - n, r.noEOF() +} + +// Next returns the next 'n' bytes in the stream. +// Unlike Peek, Next advances the reader position. +// The returned bytes point to the same +// data as the buffer, so the slice is +// only valid until the next reader method call. +// An EOF is considered an unexpected error. +// If an the returned slice is less than the +// length asked for, an error will be returned, +// and the reader position will not be incremented. +func (r *Reader) Next(n int) ([]byte, error) { + + // in case the buffer is too small + if cap(r.data) < n { + old := r.data[r.n:] + r.data = make([]byte, n+r.buffered()) + r.data = r.data[:copy(r.data, old)] + r.n = 0 + } + + // fill at least 'n' bytes + for r.buffered() < n && r.state == nil { + r.more() + } + + if r.buffered() < n { + return r.data[r.n:], r.noEOF() + } + out := r.data[r.n : r.n+n] + r.n += n + return out, nil +} + +// skipSeek uses the io.Seeker to seek forward. +// only call this function when n > r.buffered() +func (r *Reader) skipSeek(n int) (int, error) { + o := r.buffered() + // first, clear buffer + n -= o + r.n = 0 + r.data = r.data[:0] + + // then seek forward remaning bytes + i, err := r.rs.Seek(int64(n), 1) + return int(i) + o, err +} + +// Read implements `io.Reader` +func (r *Reader) Read(b []byte) (int, error) { + // if we have data in the buffer, just + // return that. + if r.buffered() != 0 { + x := copy(b, r.data[r.n:]) + r.n += x + return x, nil + } + var n int + // we have no buffered data; determine + // whether or not to buffer or call + // the underlying reader directly + if len(b) >= cap(r.data) { + n, r.state = r.r.Read(b) + } else { + r.more() + n = copy(b, r.data) + r.n = n + } + if n == 0 { + return 0, r.err() + } + return n, nil +} + +// ReadFull attempts to read len(b) bytes into +// 'b'. It returns the number of bytes read into +// 'b', and an error if it does not return len(b). +// EOF is considered an unexpected error. +func (r *Reader) ReadFull(b []byte) (int, error) { + var n int // read into b + var nn int // scratch + l := len(b) + // either read buffered data, + // or read directly for the underlying + // buffer, or fetch more buffered data. + for n < l && r.state == nil { + if r.buffered() != 0 { + nn = copy(b[n:], r.data[r.n:]) + n += nn + r.n += nn + } else if l-n > cap(r.data) { + nn, r.state = r.r.Read(b[n:]) + n += nn + } else { + r.more() + } + } + if n < l { + return n, r.noEOF() + } + return n, nil +} + +// ReadByte implements `io.ByteReader` +func (r *Reader) ReadByte() (byte, error) { + for r.buffered() < 1 && r.state == nil { + r.more() + } + if r.buffered() < 1 { + return 0, r.err() + } + b := r.data[r.n] + r.n++ + return b, nil +} + +// WriteTo implements `io.WriterTo` +func (r *Reader) WriteTo(w io.Writer) (int64, error) { + var ( + i int64 + ii int + err error + ) + // first, clear buffer + if r.buffered() > 0 { + ii, err = w.Write(r.data[r.n:]) + i += int64(ii) + if err != nil { + return i, err + } + r.data = r.data[0:0] + r.n = 0 + } + for r.state == nil { + // here we just do + // 1:1 reads and writes + r.more() + if r.buffered() > 0 { + ii, err = w.Write(r.data) + i += int64(ii) + if err != nil { + return i, err + } + r.data = r.data[0:0] + r.n = 0 + } + } + if r.state != io.EOF { + return i, r.err() + } + return i, nil +} + +func min(a int, b int) int { + if a < b { + return a + } + return b +} + +func max(a int, b int) int { + if a < b { + return b + } + return a +} diff --git a/vendor/github.com/philhofer/fwd/writer.go b/vendor/github.com/philhofer/fwd/writer.go new file mode 100644 index 00000000..2dc392a9 --- /dev/null +++ b/vendor/github.com/philhofer/fwd/writer.go @@ -0,0 +1,224 @@ +package fwd + +import "io" + +const ( + // DefaultWriterSize is the + // default write buffer size. + DefaultWriterSize = 2048 + + minWriterSize = minReaderSize +) + +// Writer is a buffered writer +type Writer struct { + w io.Writer // writer + buf []byte // 0:len(buf) is bufered data +} + +// NewWriter returns a new writer +// that writes to 'w' and has a buffer +// that is `DefaultWriterSize` bytes. +func NewWriter(w io.Writer) *Writer { + if wr, ok := w.(*Writer); ok { + return wr + } + return &Writer{ + w: w, + buf: make([]byte, 0, DefaultWriterSize), + } +} + +// NewWriterSize returns a new writer +// that writes to 'w' and has a buffer +// that is 'size' bytes. +func NewWriterSize(w io.Writer, size int) *Writer { + if wr, ok := w.(*Writer); ok && cap(wr.buf) >= size { + return wr + } + return &Writer{ + w: w, + buf: make([]byte, 0, max(size, minWriterSize)), + } +} + +// Buffered returns the number of buffered bytes +// in the reader. +func (w *Writer) Buffered() int { return len(w.buf) } + +// BufferSize returns the maximum size of the buffer. +func (w *Writer) BufferSize() int { return cap(w.buf) } + +// Flush flushes any buffered bytes +// to the underlying writer. +func (w *Writer) Flush() error { + l := len(w.buf) + if l > 0 { + n, err := w.w.Write(w.buf) + + // if we didn't write the whole + // thing, copy the unwritten + // bytes to the beginnning of the + // buffer. + if n < l && n > 0 { + w.pushback(n) + if err == nil { + err = io.ErrShortWrite + } + } + if err != nil { + return err + } + w.buf = w.buf[:0] + return nil + } + return nil +} + +// Write implements `io.Writer` +func (w *Writer) Write(p []byte) (int, error) { + c, l, ln := cap(w.buf), len(w.buf), len(p) + avail := c - l + + // requires flush + if avail < ln { + if err := w.Flush(); err != nil { + return 0, err + } + l = len(w.buf) + } + // too big to fit in buffer; + // write directly to w.w + if c < ln { + return w.w.Write(p) + } + + // grow buf slice; copy; return + w.buf = w.buf[:l+ln] + return copy(w.buf[l:], p), nil +} + +// WriteString is analogous to Write, but it takes a string. +func (w *Writer) WriteString(s string) (int, error) { + c, l, ln := cap(w.buf), len(w.buf), len(s) + avail := c - l + + // requires flush + if avail < ln { + if err := w.Flush(); err != nil { + return 0, err + } + l = len(w.buf) + } + // too big to fit in buffer; + // write directly to w.w + // + // yes, this is unsafe. *but* + // io.Writer is not allowed + // to mutate its input or + // maintain a reference to it, + // per the spec in package io. + // + // plus, if the string is really + // too big to fit in the buffer, then + // creating a copy to write it is + // expensive (and, strictly speaking, + // unnecessary) + if c < ln { + return w.w.Write(unsafestr(s)) + } + + // grow buf slice; copy; return + w.buf = w.buf[:l+ln] + return copy(w.buf[l:], s), nil +} + +// WriteByte implements `io.ByteWriter` +func (w *Writer) WriteByte(b byte) error { + if len(w.buf) == cap(w.buf) { + if err := w.Flush(); err != nil { + return err + } + } + w.buf = append(w.buf, b) + return nil +} + +// Next returns the next 'n' free bytes +// in the write buffer, flushing the writer +// as necessary. Next will return `io.ErrShortBuffer` +// if 'n' is greater than the size of the write buffer. +// Calls to 'next' increment the write position by +// the size of the returned buffer. +func (w *Writer) Next(n int) ([]byte, error) { + c, l := cap(w.buf), len(w.buf) + if n > c { + return nil, io.ErrShortBuffer + } + avail := c - l + if avail < n { + if err := w.Flush(); err != nil { + return nil, err + } + l = len(w.buf) + } + w.buf = w.buf[:l+n] + return w.buf[l:], nil +} + +// take the bytes from w.buf[n:len(w.buf)] +// and put them at the beginning of w.buf, +// and resize to the length of the copied segment. +func (w *Writer) pushback(n int) { + w.buf = w.buf[:copy(w.buf, w.buf[n:])] +} + +// ReadFrom implements `io.ReaderFrom` +func (w *Writer) ReadFrom(r io.Reader) (int64, error) { + // anticipatory flush + if err := w.Flush(); err != nil { + return 0, err + } + + w.buf = w.buf[0:cap(w.buf)] // expand buffer + + var nn int64 // written + var err error // error + var x int // read + + // 1:1 reads and writes + for err == nil { + x, err = r.Read(w.buf) + if x > 0 { + n, werr := w.w.Write(w.buf[:x]) + nn += int64(n) + + if err != nil { + if n < x && n > 0 { + w.pushback(n - x) + } + return nn, werr + } + if n < x { + w.pushback(n - x) + return nn, io.ErrShortWrite + } + } else if err == nil { + err = io.ErrNoProgress + break + } + } + if err != io.EOF { + return nn, err + } + + // we only clear here + // because we are sure + // the writes have + // succeeded. otherwise, + // we retain the data in case + // future writes succeed. + w.buf = w.buf[0:0] + + return nn, nil +} diff --git a/vendor/github.com/philhofer/fwd/writer_appengine.go b/vendor/github.com/philhofer/fwd/writer_appengine.go new file mode 100644 index 00000000..e367f393 --- /dev/null +++ b/vendor/github.com/philhofer/fwd/writer_appengine.go @@ -0,0 +1,5 @@ +// +build appengine + +package fwd + +func unsafestr(s string) []byte { return []byte(s) } diff --git a/vendor/github.com/philhofer/fwd/writer_unsafe.go b/vendor/github.com/philhofer/fwd/writer_unsafe.go new file mode 100644 index 00000000..a0bf453b --- /dev/null +++ b/vendor/github.com/philhofer/fwd/writer_unsafe.go @@ -0,0 +1,18 @@ +// +build !appengine + +package fwd + +import ( + "reflect" + "unsafe" +) + +// unsafe cast string as []byte +func unsafestr(b string) []byte { + l := len(b) + return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Len: l, + Cap: l, + Data: (*reflect.StringHeader)(unsafe.Pointer(&b)).Data, + })) +} diff --git a/vendor/github.com/spf13/afero/README.md b/vendor/github.com/spf13/afero/README.md index d9e33273..0c9b04b5 100644 --- a/vendor/github.com/spf13/afero/README.md +++ b/vendor/github.com/spf13/afero/README.md @@ -61,11 +61,11 @@ import "github.com/spf13/afero" First define a package variable and set it to a pointer to a filesystem. ```go -var AppFs afero.Fs = afero.NewMemMapFs() +var AppFs = afero.NewMemMapFs() or -var AppFs afero.Fs = afero.NewOsFs() +var AppFs = afero.NewOsFs() ``` It is important to note that if you repeat the composite literal you will be using a completely new and isolated filesystem. In the case of @@ -81,7 +81,10 @@ So if my application before had: ```go os.Open('/tmp/foo') ``` -We would replace it with a call to `AppFs.Open('/tmp/foo')`. +We would replace it with: +```go +AppFs.Open('/tmp/foo') +``` `AppFs` being the variable we defined above. @@ -166,8 +169,8 @@ f, err := afero.TempFile(fs,"", "ioutil-test") ### Calling via Afero ```go -fs := afero.NewMemMapFs -afs := &Afero{Fs: fs} +fs := afero.NewMemMapFs() +afs := &afero.Afero{Fs: fs} f, err := afs.TempFile("", "ioutil-test") ``` diff --git a/vendor/github.com/spf13/afero/appveyor.yml b/vendor/github.com/spf13/afero/appveyor.yml index 006f3153..a633ad50 100644 --- a/vendor/github.com/spf13/afero/appveyor.yml +++ b/vendor/github.com/spf13/afero/appveyor.yml @@ -12,4 +12,4 @@ build_script: go build github.com/spf13/afero test_script: -- cmd: go test -v github.com/spf13/afero +- cmd: go test -race -v github.com/spf13/afero/... diff --git a/vendor/github.com/spf13/afero/basepath.go b/vendor/github.com/spf13/afero/basepath.go index 5e4fc2ec..616ff8ff 100644 --- a/vendor/github.com/spf13/afero/basepath.go +++ b/vendor/github.com/spf13/afero/basepath.go @@ -1,7 +1,6 @@ package afero import ( - "errors" "os" "path/filepath" "runtime" @@ -9,6 +8,8 @@ import ( "time" ) +var _ Lstater = (*BasePathFs)(nil) + // The BasePathFs restricts all operations to a given path within an Fs. // The given file name to the operations on this Fs will be prepended with // the base path before calling the base Fs. @@ -22,6 +23,16 @@ type BasePathFs struct { path string } +type BasePathFile struct { + File + path string +} + +func (f *BasePathFile) Name() string { + sourcename := f.File.Name() + return strings.TrimPrefix(sourcename, filepath.Clean(f.path)) +} + func NewBasePathFs(source Fs, path string) Fs { return &BasePathFs{source: source, path: path} } @@ -30,7 +41,7 @@ func NewBasePathFs(source Fs, path string) Fs { // else the given file with the base path prepended func (b *BasePathFs) RealPath(name string) (path string, err error) { if err := validateBasePathName(name); err != nil { - return "", err + return name, err } bpath := filepath.Clean(b.path) @@ -52,7 +63,7 @@ func validateBasePathName(name string) error { // On Windows a common mistake would be to provide an absolute OS path // We could strip out the base part, but that would not be very portable. if filepath.IsAbs(name) { - return &os.PathError{Op: "realPath", Path: name, Err: errors.New("got a real OS path instead of a virtual")} + return os.ErrNotExist } return nil @@ -111,14 +122,22 @@ func (b *BasePathFs) OpenFile(name string, flag int, mode os.FileMode) (f File, if name, err = b.RealPath(name); err != nil { return nil, &os.PathError{Op: "openfile", Path: name, Err: err} } - return b.source.OpenFile(name, flag, mode) + sourcef, err := b.source.OpenFile(name, flag, mode) + if err != nil { + return nil, err + } + return &BasePathFile{sourcef, b.path}, nil } func (b *BasePathFs) Open(name string) (f File, err error) { if name, err = b.RealPath(name); err != nil { return nil, &os.PathError{Op: "open", Path: name, Err: err} } - return b.source.Open(name) + sourcef, err := b.source.Open(name) + if err != nil { + return nil, err + } + return &BasePathFile{File: sourcef, path: b.path}, nil } func (b *BasePathFs) Mkdir(name string, mode os.FileMode) (err error) { @@ -139,7 +158,23 @@ func (b *BasePathFs) Create(name string) (f File, err error) { if name, err = b.RealPath(name); err != nil { return nil, &os.PathError{Op: "create", Path: name, Err: err} } - return b.source.Create(name) + sourcef, err := b.source.Create(name) + if err != nil { + return nil, err + } + return &BasePathFile{File: sourcef, path: b.path}, nil +} + +func (b *BasePathFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + name, err := b.RealPath(name) + if err != nil { + return nil, false, &os.PathError{Op: "lstat", Path: name, Err: err} + } + if lstater, ok := b.source.(Lstater); ok { + return lstater.LstatIfPossible(name) + } + fi, err := b.source.Stat(name) + return fi, false, err } // vim: ts=4 sw=4 noexpandtab nolist syn=go diff --git a/vendor/github.com/spf13/afero/cacheOnReadFs.go b/vendor/github.com/spf13/afero/cacheOnReadFs.go index e54a4f8b..29a26c67 100644 --- a/vendor/github.com/spf13/afero/cacheOnReadFs.go +++ b/vendor/github.com/spf13/afero/cacheOnReadFs.go @@ -64,15 +64,10 @@ func (u *CacheOnReadFs) cacheStatus(name string) (state cacheState, fi os.FileIn return cacheHit, lfi, nil } - if err == syscall.ENOENT { + if err == syscall.ENOENT || os.IsNotExist(err) { return cacheMiss, nil, nil } - var ok bool - if err, ok = err.(*os.PathError); ok { - if err == os.ErrNotExist { - return cacheMiss, nil, nil - } - } + return cacheMiss, nil, err } @@ -210,7 +205,7 @@ func (u *CacheOnReadFs) OpenFile(name string, flag int, perm os.FileMode) (File, bfi.Close() // oops, what if O_TRUNC was set and file opening in the layer failed...? return nil, err } - return &UnionFile{base: bfi, layer: lfi}, nil + return &UnionFile{Base: bfi, Layer: lfi}, nil } return u.layer.OpenFile(name, flag, perm) } @@ -256,7 +251,7 @@ func (u *CacheOnReadFs) Open(name string) (File, error) { if err != nil && bfile == nil { return nil, err } - return &UnionFile{base: bfile, layer: lfile}, nil + return &UnionFile{Base: bfile, Layer: lfile}, nil } func (u *CacheOnReadFs) Mkdir(name string, perm os.FileMode) error { @@ -291,5 +286,5 @@ func (u *CacheOnReadFs) Create(name string) (File, error) { bfh.Close() return nil, err } - return &UnionFile{base: bfh, layer: lfh}, nil + return &UnionFile{Base: bfh, Layer: lfh}, nil } diff --git a/vendor/github.com/spf13/afero/copyOnWriteFs.go b/vendor/github.com/spf13/afero/copyOnWriteFs.go index ed692ae9..9aef3979 100644 --- a/vendor/github.com/spf13/afero/copyOnWriteFs.go +++ b/vendor/github.com/spf13/afero/copyOnWriteFs.go @@ -8,6 +8,8 @@ import ( "time" ) +var _ Lstater = (*CopyOnWriteFs)(nil) + // The CopyOnWriteFs is a union filesystem: a read only base file system with // a possibly writeable layer on top. Changes to the file system will only // be made in the overlay: Changing an existing file in the base layer which @@ -76,18 +78,55 @@ func (u *CopyOnWriteFs) Chmod(name string, mode os.FileMode) error { func (u *CopyOnWriteFs) Stat(name string) (os.FileInfo, error) { fi, err := u.layer.Stat(name) if err != nil { - origErr := err - if e, ok := err.(*os.PathError); ok { - err = e.Err - } - if err == syscall.ENOENT || err == syscall.ENOTDIR { + isNotExist := u.isNotExist(err) + if isNotExist { return u.base.Stat(name) } - return nil, origErr + return nil, err } return fi, nil } +func (u *CopyOnWriteFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + llayer, ok1 := u.layer.(Lstater) + lbase, ok2 := u.base.(Lstater) + + if ok1 { + fi, b, err := llayer.LstatIfPossible(name) + if err == nil { + return fi, b, nil + } + + if !u.isNotExist(err) { + return nil, b, err + } + } + + if ok2 { + fi, b, err := lbase.LstatIfPossible(name) + if err == nil { + return fi, b, nil + } + if !u.isNotExist(err) { + return nil, b, err + } + } + + fi, err := u.Stat(name) + + return fi, false, err +} + +func (u *CopyOnWriteFs) isNotExist(err error) bool { + if e, ok := err.(*os.PathError); ok { + err = e.Err + } + if err == os.ErrNotExist || err == syscall.ENOENT || err == syscall.ENOTDIR { + return true + } + return false +} + // Renaming files present only in the base layer is not permitted func (u *CopyOnWriteFs) Rename(oldname, newname string) error { b, err := u.isBaseFile(oldname) @@ -219,7 +258,7 @@ func (u *CopyOnWriteFs) Open(name string) (File, error) { return nil, fmt.Errorf("BaseErr: %v\nOverlayErr: %v", bErr, lErr) } - return &UnionFile{base: bfile, layer: lfile}, nil + return &UnionFile{Base: bfile, Layer: lfile}, nil } func (u *CopyOnWriteFs) Mkdir(name string, perm os.FileMode) error { diff --git a/vendor/github.com/spf13/afero/lstater.go b/vendor/github.com/spf13/afero/lstater.go new file mode 100644 index 00000000..89c1bfc0 --- /dev/null +++ b/vendor/github.com/spf13/afero/lstater.go @@ -0,0 +1,27 @@ +// Copyright © 2018 Steve Francia . +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "os" +) + +// Lstater is an optional interface in Afero. It is only implemented by the +// filesystems saying so. +// It will call Lstat if the filesystem iself is, or it delegates to, the os filesystem. +// Else it will call Stat. +// In addtion to the FileInfo, it will return a boolean telling whether Lstat was called or not. +type Lstater interface { + LstatIfPossible(name string) (os.FileInfo, bool, error) +} diff --git a/vendor/github.com/spf13/afero/match.go b/vendor/github.com/spf13/afero/match.go new file mode 100644 index 00000000..c18a87fb --- /dev/null +++ b/vendor/github.com/spf13/afero/match.go @@ -0,0 +1,110 @@ +// Copyright © 2014 Steve Francia . +// Copyright 2009 The Go Authors. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "path/filepath" + "sort" + "strings" +) + +// Glob returns the names of all files matching pattern or nil +// if there is no matching file. The syntax of patterns is the same +// as in Match. The pattern may describe hierarchical names such as +// /usr/*/bin/ed (assuming the Separator is '/'). +// +// Glob ignores file system errors such as I/O errors reading directories. +// The only possible returned error is ErrBadPattern, when pattern +// is malformed. +// +// This was adapted from (http://golang.org/pkg/path/filepath) and uses several +// built-ins from that package. +func Glob(fs Fs, pattern string) (matches []string, err error) { + if !hasMeta(pattern) { + // Lstat not supported by a ll filesystems. + if _, err = lstatIfPossible(fs, pattern); err != nil { + return nil, nil + } + return []string{pattern}, nil + } + + dir, file := filepath.Split(pattern) + switch dir { + case "": + dir = "." + case string(filepath.Separator): + // nothing + default: + dir = dir[0 : len(dir)-1] // chop off trailing separator + } + + if !hasMeta(dir) { + return glob(fs, dir, file, nil) + } + + var m []string + m, err = Glob(fs, dir) + if err != nil { + return + } + for _, d := range m { + matches, err = glob(fs, d, file, matches) + if err != nil { + return + } + } + return +} + +// glob searches for files matching pattern in the directory dir +// and appends them to matches. If the directory cannot be +// opened, it returns the existing matches. New matches are +// added in lexicographical order. +func glob(fs Fs, dir, pattern string, matches []string) (m []string, e error) { + m = matches + fi, err := fs.Stat(dir) + if err != nil { + return + } + if !fi.IsDir() { + return + } + d, err := fs.Open(dir) + if err != nil { + return + } + defer d.Close() + + names, _ := d.Readdirnames(-1) + sort.Strings(names) + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + if matched { + m = append(m, filepath.Join(dir, n)) + } + } + return +} + +// hasMeta reports whether path contains any of the magic characters +// recognized by Match. +func hasMeta(path string) bool { + // TODO(niemeyer): Should other magic characters be added here? + return strings.IndexAny(path, "*?[") >= 0 +} diff --git a/vendor/github.com/spf13/afero/mem/file.go b/vendor/github.com/spf13/afero/mem/file.go index e41e0123..885e5542 100644 --- a/vendor/github.com/spf13/afero/mem/file.go +++ b/vendor/github.com/spf13/afero/mem/file.go @@ -74,14 +74,24 @@ func CreateDir(name string) *FileData { } func ChangeFileName(f *FileData, newname string) { + f.Lock() f.name = newname + f.Unlock() } func SetMode(f *FileData, mode os.FileMode) { + f.Lock() f.mode = mode + f.Unlock() } func SetModTime(f *FileData, mtime time.Time) { + f.Lock() + setModTime(f, mtime) + f.Unlock() +} + +func setModTime(f *FileData, mtime time.Time) { f.modtime = mtime } @@ -102,7 +112,7 @@ func (f *File) Close() error { f.fileData.Lock() f.closed = true if !f.readOnly { - SetModTime(f.fileData, time.Now()) + setModTime(f.fileData, time.Now()) } f.fileData.Unlock() return nil @@ -166,6 +176,9 @@ func (f *File) Read(b []byte) (n int, err error) { if len(b) > 0 && int(f.at) == len(f.fileData.data) { return 0, io.EOF } + if int(f.at) > len(f.fileData.data) { + return 0, io.ErrUnexpectedEOF + } if len(f.fileData.data)-int(f.at) >= len(b) { n = len(b) } else { @@ -197,7 +210,7 @@ func (f *File) Truncate(size int64) error { } else { f.fileData.data = f.fileData.data[0:size] } - SetModTime(f.fileData, time.Now()) + setModTime(f.fileData, time.Now()) return nil } @@ -236,7 +249,7 @@ func (f *File) Write(b []byte) (n int, err error) { f.fileData.data = append(f.fileData.data[:cur], b...) f.fileData.data = append(f.fileData.data, tail...) } - SetModTime(f.fileData, time.Now()) + setModTime(f.fileData, time.Now()) atomic.StoreInt64(&f.at, int64(len(f.fileData.data))) return @@ -261,17 +274,33 @@ type FileInfo struct { // Implements os.FileInfo func (s *FileInfo) Name() string { + s.Lock() _, name := filepath.Split(s.name) + s.Unlock() return name } -func (s *FileInfo) Mode() os.FileMode { return s.mode } -func (s *FileInfo) ModTime() time.Time { return s.modtime } -func (s *FileInfo) IsDir() bool { return s.dir } -func (s *FileInfo) Sys() interface{} { return nil } +func (s *FileInfo) Mode() os.FileMode { + s.Lock() + defer s.Unlock() + return s.mode +} +func (s *FileInfo) ModTime() time.Time { + s.Lock() + defer s.Unlock() + return s.modtime +} +func (s *FileInfo) IsDir() bool { + s.Lock() + defer s.Unlock() + return s.dir +} +func (s *FileInfo) Sys() interface{} { return nil } func (s *FileInfo) Size() int64 { if s.IsDir() { return int64(42) } + s.Lock() + defer s.Unlock() return int64(len(s.data)) } diff --git a/vendor/github.com/spf13/afero/memmap.go b/vendor/github.com/spf13/afero/memmap.go index 767ac1d5..09498e70 100644 --- a/vendor/github.com/spf13/afero/memmap.go +++ b/vendor/github.com/spf13/afero/memmap.go @@ -66,7 +66,10 @@ func (m *MemMapFs) unRegisterWithParent(fileName string) error { if parent == nil { log.Panic("parent of ", f.Name(), " is nil") } + + parent.Lock() mem.RemoveFromMemDir(parent, f) + parent.Unlock() return nil } @@ -99,8 +102,10 @@ func (m *MemMapFs) registerWithParent(f *mem.FileData) { } } + parent.Lock() mem.InitializeDir(parent) mem.AddToMemDir(parent, f) + parent.Unlock() } func (m *MemMapFs) lockfreeMkdir(name string, perm os.FileMode) error { @@ -136,7 +141,7 @@ func (m *MemMapFs) Mkdir(name string, perm os.FileMode) error { m.registerWithParent(item) m.mu.Unlock() - m.Chmod(name, perm) + m.Chmod(name, perm|os.ModeDir) return nil } @@ -146,9 +151,8 @@ func (m *MemMapFs) MkdirAll(path string, perm os.FileMode) error { if err != nil { if err.(*os.PathError).Err == ErrFileExists { return nil - } else { - return err } + return err } return nil } diff --git a/vendor/github.com/spf13/afero/os.go b/vendor/github.com/spf13/afero/os.go index 6b8bce1c..13cc1b84 100644 --- a/vendor/github.com/spf13/afero/os.go +++ b/vendor/github.com/spf13/afero/os.go @@ -19,6 +19,8 @@ import ( "time" ) +var _ Lstater = (*OsFs)(nil) + // OsFs is a Fs implementation that uses functions provided by the os package. // // For details in any method, check the documentation of the os package @@ -92,3 +94,8 @@ func (OsFs) Chmod(name string, mode os.FileMode) error { func (OsFs) Chtimes(name string, atime time.Time, mtime time.Time) error { return os.Chtimes(name, atime, mtime) } + +func (OsFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + fi, err := os.Lstat(name) + return fi, true, err +} diff --git a/vendor/github.com/spf13/afero/path.go b/vendor/github.com/spf13/afero/path.go index 1d90e46d..18f60a0f 100644 --- a/vendor/github.com/spf13/afero/path.go +++ b/vendor/github.com/spf13/afero/path.go @@ -60,7 +60,7 @@ func walk(fs Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error for _, name := range names { filename := filepath.Join(path, name) - fileInfo, err := lstatIfOs(fs, filename) + fileInfo, err := lstatIfPossible(fs, filename) if err != nil { if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { return err @@ -77,15 +77,13 @@ func walk(fs Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error return nil } -// if the filesystem is OsFs use Lstat, else use fs.Stat -func lstatIfOs(fs Fs, path string) (info os.FileInfo, err error) { - _, ok := fs.(*OsFs) - if ok { - info, err = os.Lstat(path) - } else { - info, err = fs.Stat(path) +// if the filesystem supports it, use Lstat, else use fs.Stat +func lstatIfPossible(fs Fs, path string) (os.FileInfo, error) { + if lfs, ok := fs.(Lstater); ok { + fi, _, err := lfs.LstatIfPossible(path) + return fi, err } - return + return fs.Stat(path) } // Walk walks the file tree rooted at root, calling walkFn for each file or @@ -100,7 +98,7 @@ func (a Afero) Walk(root string, walkFn filepath.WalkFunc) error { } func Walk(fs Fs, root string, walkFn filepath.WalkFunc) error { - info, err := lstatIfOs(fs, root) + info, err := lstatIfPossible(fs, root) if err != nil { return walkFn(root, nil, err) } diff --git a/vendor/github.com/spf13/afero/readonlyfs.go b/vendor/github.com/spf13/afero/readonlyfs.go index f1fa55bc..c6376ec3 100644 --- a/vendor/github.com/spf13/afero/readonlyfs.go +++ b/vendor/github.com/spf13/afero/readonlyfs.go @@ -6,6 +6,8 @@ import ( "time" ) +var _ Lstater = (*ReadOnlyFs)(nil) + type ReadOnlyFs struct { source Fs } @@ -34,6 +36,14 @@ func (r *ReadOnlyFs) Stat(name string) (os.FileInfo, error) { return r.source.Stat(name) } +func (r *ReadOnlyFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { + if lsf, ok := r.source.(Lstater); ok { + return lsf.LstatIfPossible(name) + } + fi, err := r.Stat(name) + return fi, false, err +} + func (r *ReadOnlyFs) Rename(o, n string) error { return syscall.EPERM } diff --git a/vendor/github.com/spf13/afero/unionFile.go b/vendor/github.com/spf13/afero/unionFile.go index 99f9e5db..1e78f7d1 100644 --- a/vendor/github.com/spf13/afero/unionFile.go +++ b/vendor/github.com/spf13/afero/unionFile.go @@ -21,32 +21,33 @@ import ( // successful read in the overlay will move the cursor position in the base layer // by the number of bytes read. type UnionFile struct { - base File - layer File - off int - files []os.FileInfo + Base File + Layer File + Merger DirsMerger + off int + files []os.FileInfo } func (f *UnionFile) Close() error { // first close base, so we have a newer timestamp in the overlay. If we'd close // the overlay first, we'd get a cacheStale the next time we access this file // -> cache would be useless ;-) - if f.base != nil { - f.base.Close() + if f.Base != nil { + f.Base.Close() } - if f.layer != nil { - return f.layer.Close() + if f.Layer != nil { + return f.Layer.Close() } return BADFD } func (f *UnionFile) Read(s []byte) (int, error) { - if f.layer != nil { - n, err := f.layer.Read(s) - if (err == nil || err == io.EOF) && f.base != nil { + if f.Layer != nil { + n, err := f.Layer.Read(s) + if (err == nil || err == io.EOF) && f.Base != nil { // advance the file position also in the base file, the next // call may be a write at this position (or a seek with SEEK_CUR) - if _, seekErr := f.base.Seek(int64(n), os.SEEK_CUR); seekErr != nil { + if _, seekErr := f.Base.Seek(int64(n), os.SEEK_CUR); seekErr != nil { // only overwrite err in case the seek fails: we need to // report an eventual io.EOF to the caller err = seekErr @@ -54,105 +55,135 @@ func (f *UnionFile) Read(s []byte) (int, error) { } return n, err } - if f.base != nil { - return f.base.Read(s) + if f.Base != nil { + return f.Base.Read(s) } return 0, BADFD } func (f *UnionFile) ReadAt(s []byte, o int64) (int, error) { - if f.layer != nil { - n, err := f.layer.ReadAt(s, o) - if (err == nil || err == io.EOF) && f.base != nil { - _, err = f.base.Seek(o+int64(n), os.SEEK_SET) + if f.Layer != nil { + n, err := f.Layer.ReadAt(s, o) + if (err == nil || err == io.EOF) && f.Base != nil { + _, err = f.Base.Seek(o+int64(n), os.SEEK_SET) } return n, err } - if f.base != nil { - return f.base.ReadAt(s, o) + if f.Base != nil { + return f.Base.ReadAt(s, o) } return 0, BADFD } func (f *UnionFile) Seek(o int64, w int) (pos int64, err error) { - if f.layer != nil { - pos, err = f.layer.Seek(o, w) - if (err == nil || err == io.EOF) && f.base != nil { - _, err = f.base.Seek(o, w) + if f.Layer != nil { + pos, err = f.Layer.Seek(o, w) + if (err == nil || err == io.EOF) && f.Base != nil { + _, err = f.Base.Seek(o, w) } return pos, err } - if f.base != nil { - return f.base.Seek(o, w) + if f.Base != nil { + return f.Base.Seek(o, w) } return 0, BADFD } func (f *UnionFile) Write(s []byte) (n int, err error) { - if f.layer != nil { - n, err = f.layer.Write(s) - if err == nil && f.base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark? - _, err = f.base.Write(s) + if f.Layer != nil { + n, err = f.Layer.Write(s) + if err == nil && f.Base != nil { // hmm, do we have fixed size files where a write may hit the EOF mark? + _, err = f.Base.Write(s) } return n, err } - if f.base != nil { - return f.base.Write(s) + if f.Base != nil { + return f.Base.Write(s) } return 0, BADFD } func (f *UnionFile) WriteAt(s []byte, o int64) (n int, err error) { - if f.layer != nil { - n, err = f.layer.WriteAt(s, o) - if err == nil && f.base != nil { - _, err = f.base.WriteAt(s, o) + if f.Layer != nil { + n, err = f.Layer.WriteAt(s, o) + if err == nil && f.Base != nil { + _, err = f.Base.WriteAt(s, o) } return n, err } - if f.base != nil { - return f.base.WriteAt(s, o) + if f.Base != nil { + return f.Base.WriteAt(s, o) } return 0, BADFD } func (f *UnionFile) Name() string { - if f.layer != nil { - return f.layer.Name() + if f.Layer != nil { + return f.Layer.Name() } - return f.base.Name() + return f.Base.Name() +} + +// DirsMerger is how UnionFile weaves two directories together. +// It takes the FileInfo slices from the layer and the base and returns a +// single view. +type DirsMerger func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) + +var defaultUnionMergeDirsFn = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { + var files = make(map[string]os.FileInfo) + + for _, fi := range lofi { + files[fi.Name()] = fi + } + + for _, fi := range bofi { + if _, exists := files[fi.Name()]; !exists { + files[fi.Name()] = fi + } + } + + rfi := make([]os.FileInfo, len(files)) + + i := 0 + for _, fi := range files { + rfi[i] = fi + i++ + } + + return rfi, nil + } // Readdir will weave the two directories together and // return a single view of the overlayed directories func (f *UnionFile) Readdir(c int) (ofi []os.FileInfo, err error) { + var merge DirsMerger = f.Merger + if merge == nil { + merge = defaultUnionMergeDirsFn + } + if f.off == 0 { - var files = make(map[string]os.FileInfo) - var rfi []os.FileInfo - if f.layer != nil { - rfi, err = f.layer.Readdir(-1) + var lfi []os.FileInfo + if f.Layer != nil { + lfi, err = f.Layer.Readdir(-1) if err != nil { return nil, err } - for _, fi := range rfi { - files[fi.Name()] = fi - } } - if f.base != nil { - rfi, err = f.base.Readdir(-1) + var bfi []os.FileInfo + if f.Base != nil { + bfi, err = f.Base.Readdir(-1) if err != nil { return nil, err } - for _, fi := range rfi { - if _, exists := files[fi.Name()]; !exists { - files[fi.Name()] = fi - } - } + } - for _, fi := range files { - f.files = append(f.files, fi) + merged, err := merge(lfi, bfi) + if err != nil { + return nil, err } + f.files = append(f.files, merged...) } if c == -1 { return f.files[f.off:], nil @@ -174,53 +205,53 @@ func (f *UnionFile) Readdirnames(c int) ([]string, error) { } func (f *UnionFile) Stat() (os.FileInfo, error) { - if f.layer != nil { - return f.layer.Stat() + if f.Layer != nil { + return f.Layer.Stat() } - if f.base != nil { - return f.base.Stat() + if f.Base != nil { + return f.Base.Stat() } return nil, BADFD } func (f *UnionFile) Sync() (err error) { - if f.layer != nil { - err = f.layer.Sync() - if err == nil && f.base != nil { - err = f.base.Sync() + if f.Layer != nil { + err = f.Layer.Sync() + if err == nil && f.Base != nil { + err = f.Base.Sync() } return err } - if f.base != nil { - return f.base.Sync() + if f.Base != nil { + return f.Base.Sync() } return BADFD } func (f *UnionFile) Truncate(s int64) (err error) { - if f.layer != nil { - err = f.layer.Truncate(s) - if err == nil && f.base != nil { - err = f.base.Truncate(s) + if f.Layer != nil { + err = f.Layer.Truncate(s) + if err == nil && f.Base != nil { + err = f.Base.Truncate(s) } return err } - if f.base != nil { - return f.base.Truncate(s) + if f.Base != nil { + return f.Base.Truncate(s) } return BADFD } func (f *UnionFile) WriteString(s string) (n int, err error) { - if f.layer != nil { - n, err = f.layer.WriteString(s) - if err == nil && f.base != nil { - _, err = f.base.WriteString(s) + if f.Layer != nil { + n, err = f.Layer.WriteString(s) + if err == nil && f.Base != nil { + _, err = f.Base.WriteString(s) } return n, err } - if f.base != nil { - return f.base.WriteString(s) + if f.Base != nil { + return f.Base.WriteString(s) } return 0, BADFD } diff --git a/vendor/github.com/spf13/afero/util.go b/vendor/github.com/spf13/afero/util.go index 2f44e6a1..4f253f48 100644 --- a/vendor/github.com/spf13/afero/util.go +++ b/vendor/github.com/spf13/afero/util.go @@ -20,7 +20,6 @@ import ( "bytes" "fmt" "io" - "log" "os" "path/filepath" "strings" @@ -46,7 +45,7 @@ func WriteReader(fs Fs, path string, r io.Reader) (err error) { err = fs.MkdirAll(ospath, 0777) // rwx, rw, r if err != nil { if err != os.ErrExist { - log.Panicln(err) + return err } } } @@ -157,7 +156,7 @@ func UnicodeSanitize(s string) string { return string(target) } -// Transform characters with accents into plan forms +// Transform characters with accents into plain forms. func NeuterAccents(s string) string { t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC) result, _, _ := transform.String(t, string(s)) diff --git a/vendor/github.com/spf13/cast/caste.go b/vendor/github.com/spf13/cast/caste.go index 81511fe5..4fe19289 100644 --- a/vendor/github.com/spf13/cast/caste.go +++ b/vendor/github.com/spf13/cast/caste.go @@ -6,6 +6,7 @@ package cast import ( + "encoding/json" "errors" "fmt" "html/template" @@ -872,6 +873,9 @@ func ToStringMapStringE(i interface{}) (map[string]string, error) { m[ToString(k)] = ToString(val) } return m, nil + case string: + err := jsonStringToObject(v, &m) + return m, err default: return m, fmt.Errorf("unable to cast %#v of type %T to map[string]string", i, i) } @@ -932,6 +936,9 @@ func ToStringMapStringSliceE(i interface{}) (map[string][]string, error) { } m[key] = value } + case string: + err := jsonStringToObject(v, &m) + return m, err default: return m, fmt.Errorf("unable to cast %#v of type %T to map[string][]string", i, i) } @@ -955,6 +962,9 @@ func ToStringMapBoolE(i interface{}) (map[string]bool, error) { return m, nil case map[string]bool: return v, nil + case string: + err := jsonStringToObject(v, &m) + return m, err default: return m, fmt.Errorf("unable to cast %#v of type %T to map[string]bool", i, i) } @@ -972,6 +982,9 @@ func ToStringMapE(i interface{}) (map[string]interface{}, error) { return m, nil case map[string]interface{}: return v, nil + case string: + err := jsonStringToObject(v, &m) + return m, err default: return m, fmt.Errorf("unable to cast %#v of type %T to map[string]interface{}", i, i) } @@ -1144,3 +1157,10 @@ func parseDateWith(s string, dates []string) (d time.Time, e error) { } return d, fmt.Errorf("unable to parse date: %s", s) } + +// jsonStringToObject attempts to unmarshall a string as JSON into +// the object passed as pointer. +func jsonStringToObject(s string, v interface{}) error { + data := []byte(s) + return json.Unmarshal(data, v) +} diff --git a/vendor/github.com/spf13/cobra/README.md b/vendor/github.com/spf13/cobra/README.md index 0d5bbfb1..851fcc08 100644 --- a/vendor/github.com/spf13/cobra/README.md +++ b/vendor/github.com/spf13/cobra/README.md @@ -8,7 +8,7 @@ Many of the most widely used Go projects are built using Cobra including: * [Hugo](http://gohugo.io) * [rkt](https://github.com/coreos/rkt) * [etcd](https://github.com/coreos/etcd) -* [Docker](https://github.com/docker/docker) +* [Moby (former Docker)](https://github.com/moby/moby) * [Docker (distribution)](https://github.com/docker/distribution) * [OpenShift](https://www.openshift.com/) * [Delve](https://github.com/derekparker/delve) @@ -16,16 +16,37 @@ Many of the most widely used Go projects are built using Cobra including: * [CockroachDB](http://www.cockroachlabs.com/) * [Bleve](http://www.blevesearch.com/) * [ProjectAtomic (enterprise)](http://www.projectatomic.io/) -* [Parse (CLI)](https://parse.com/) * [GiantSwarm's swarm](https://github.com/giantswarm/cli) * [Nanobox](https://github.com/nanobox-io/nanobox)/[Nanopack](https://github.com/nanopack) - +* [rclone](http://rclone.org/) +* [nehm](https://github.com/bogem/nehm) +* [Pouch](https://github.com/alibaba/pouch) [![Build Status](https://travis-ci.org/spf13/cobra.svg "Travis CI status")](https://travis-ci.org/spf13/cobra) [![CircleCI status](https://circleci.com/gh/spf13/cobra.png?circle-token=:circle-token "CircleCI status")](https://circleci.com/gh/spf13/cobra) -[![GoDoc](https://godoc.org/github.com/spf13/cobra?status.svg)](https://godoc.org/github.com/spf13/cobra) +[![GoDoc](https://godoc.org/github.com/spf13/cobra?status.svg)](https://godoc.org/github.com/spf13/cobra) -![cobra](https://cloud.githubusercontent.com/assets/173412/10911369/84832a8e-8212-11e5-9f82-cc96660a4794.gif) +# Table of Contents + +- [Overview](#overview) +- [Concepts](#concepts) + * [Commands](#commands) + * [Flags](#flags) +- [Installing](#installing) +- [Getting Started](#getting-started) + * [Using the Cobra Generator](#using-the-cobra-generator) + * [Using the Cobra Library](#using-the-cobra-library) + * [Working with Flags](#working-with-flags) + * [Positional and Custom Arguments](#positional-and-custom-arguments) + * [Example](#example) + * [Help Command](#help-command) + * [Usage Message](#usage-message) + * [PreRun and PostRun Hooks](#prerun-and-postrun-hooks) + * [Suggestions when "unknown command" happens](#suggestions-when-unknown-command-happens) + * [Generating documentation for your command](#generating-documentation-for-your-command) + * [Generating bash completions](#generating-bash-completions) +- [Contributing](#contributing) +- [License](#license) # Overview @@ -40,27 +61,16 @@ Cobra provides: * Fully POSIX-compliant flags (including short & long versions) * Nested subcommands * Global, local and cascading flags -* Easy generation of applications & commands with `cobra create appname` & `cobra add cmdname` +* Easy generation of applications & commands with `cobra init appname` & `cobra add cmdname` * Intelligent suggestions (`app srver`... did you mean `app server`?) * Automatic help generation for commands and flags -* Automatic detailed help for `app help [command]` * Automatic help flag recognition of `-h`, `--help`, etc. * Automatically generated bash autocomplete for your application * Automatically generated man pages for your application * Command aliases so you can change things without breaking them -* The flexibilty to define your own help, usage, etc. +* The flexibility to define your own help, usage, etc. * Optional tight integration with [viper](http://github.com/spf13/viper) for 12-factor apps -Cobra has an exceptionally clean interface and simple design without needless -constructors or initialization methods. - -Applications built with Cobra commands are designed to be as user-friendly as -possible. Flags can be placed before or after the command (as long as a -confusing space isn’t provided). Both short and long flags can be used. A -command need not even be fully typed. Help is automatically generated and -available for the application or for a specific command using either the help -command or the `--help` flag. - # Concepts Cobra is built on a structure of commands, arguments & flags. @@ -79,11 +89,11 @@ A few good real world examples may better illustrate this point. In the following example, 'server' is a command, and 'port' is a flag: - > hugo server --port=1313 + hugo server --port=1313 In this command we are telling Git to clone the url bare. - > git clone URL --bare + git clone URL --bare ## Commands @@ -93,20 +103,11 @@ have children commands and optionally run an action. In the example above, 'server' is the command. -A Command has the following structure: - -```go -type Command struct { - Use string // The one-line usage message. - Short string // The short description shown in the 'help' output. - Long string // The long message shown in the 'help ' output. - Run func(cmd *Command, args []string) // Run runs the command. -} -``` +[More about cobra.Command](https://godoc.org/github.com/spf13/cobra#Command) ## Flags -A Flag is a way to modify the behavior of a command. Cobra supports +A flag is a way to modify the behavior of a command. Cobra supports fully POSIX-compliant flags as well as the Go [flag package](https://golang.org/pkg/flag/). A Cobra command can define flags that persist through to children commands and flags that are only available to that command. @@ -114,23 +115,15 @@ and flags that are only available to that command. In the example above, 'port' is the flag. Flag functionality is provided by the [pflag -library](https://github.com/ogier/pflag), a fork of the flag standard library +library](https://github.com/spf13/pflag), a fork of the flag standard library which maintains the same interface while adding POSIX compliance. -## Usage - -Cobra works by creating a set of commands and then organizing them into a tree. -The tree defines the structure of the application. - -Once each command is defined with its corresponding flags, then the -tree is assigned to the commander which is finally executed. - # Installing Using Cobra is easy. First, use `go get` to install the latest version -of the library. This command will install the `cobra` generator executible -along with the library: +of the library. This command will install the `cobra` generator executable +along with the library and its dependencies: - > go get -v github.com/spf13/cobra/cobra + go get -u github.com/spf13/cobra/cobra Next, include Cobra in your application: @@ -140,8 +133,8 @@ import "github.com/spf13/cobra" # Getting Started -While you are welcome to provide your own organization, typically a Cobra based -application will follow the following organizational structure. +While you are welcome to provide your own organization, typically a Cobra-based +application will follow the following organizational structure: ``` ▾ appName/ @@ -153,23 +146,20 @@ application will follow the following organizational structure. main.go ``` -In a Cobra app, typically the main.go file is very bare. It serves, one purpose, to initialize Cobra. +In a Cobra app, typically the main.go file is very bare. It serves one purpose: initializing Cobra. ```go package main import ( - "fmt" - "os" + "fmt" + "os" - "{pathToYourApp}/cmd" + "{pathToYourApp}/cmd" ) func main() { - if err := cmd.RootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } + cmd.Execute() } ``` @@ -178,134 +168,89 @@ func main() { Cobra provides its own program that will create your application and add any commands you want. It's the easiest way to incorporate Cobra into your application. -In order to use the cobra command, compile it using the following command: +[Here](https://github.com/spf13/cobra/blob/master/cobra/README.md) you can find more information about it. - > go install github.com/spf13/cobra/cobra +## Using the Cobra Library -This will create the cobra executable under your go path bin directory! - -### cobra init - -The `cobra init [yourApp]` command will create your initial application code -for you. It is a very powerful application that will populate your program with -the right structure so you can immediately enjoy all the benefits of Cobra. It -will also automatically apply the license you specify to your application. - -Cobra init is pretty smart. You can provide it a full path, or simply a path -similar to what is expected in the import. - -``` -cobra init github.com/spf13/newAppName -``` - -### cobra add - -Once an application is initialized Cobra can create additional commands for you. -Let's say you created an app and you wanted the following commands for it: - -* app serve -* app config -* app config create - -In your project directory (where your main.go file is) you would run the following: - -``` -cobra add serve -cobra add config -cobra add create -p 'configCmd' -``` - -Once you have run these three commands you would have an app structure that would look like: - -``` - ▾ app/ - ▾ cmd/ - serve.go - config.go - create.go - main.go -``` - -at this point you can run `go run main.go` and it would run your app. `go run -main.go serve`, `go run main.go config`, `go run main.go config create` along -with `go run main.go help serve`, etc would all work. - -Obviously you haven't added your own code to these yet, the commands are ready -for you to give them their tasks. Have fun. - -### Configuring the cobra generator - -The cobra generator will be easier to use if you provide a simple configuration -file which will help you eliminate providing a bunch of repeated information in -flags over and over. - -An example ~/.cobra.yaml file: - -```yaml -author: Steve Francia -license: MIT -``` - -You can specify no license by setting `license` to `none` or you can specify -a custom license: - -```yaml -license: - header: This file is part of {{ .appName }}. - text: | - {{ .copyright }} - - This is my license. There are many like it, but this one is mine. - My license is my best friend. It is my life. I must master it as I must - master my life. -``` - -## Manually implementing Cobra - -To manually implement cobra you need to create a bare main.go file and a RootCmd file. +To manually implement Cobra you need to create a bare main.go file and a rootCmd file. You will optionally provide additional commands as you see fit. -### Create the root command - -The root command represents your binary itself. - - -#### Manually create rootCmd +### Create rootCmd Cobra doesn't require any special constructors. Simply create your commands. Ideally you place this in app/cmd/root.go: ```go -var RootCmd = &cobra.Command{ - Use: "hugo", - Short: "Hugo is a very fast static site generator", - Long: `A Fast and Flexible Static Site Generator built with +var rootCmd = &cobra.Command{ + Use: "hugo", + Short: "Hugo is a very fast static site generator", + Long: `A Fast and Flexible Static Site Generator built with love by spf13 and friends in Go. Complete documentation is available at http://hugo.spf13.com`, - Run: func(cmd *cobra.Command, args []string) { - // Do Stuff Here - }, + Run: func(cmd *cobra.Command, args []string) { + // Do Stuff Here + }, +} + +func Execute() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } } ``` You will additionally define flags and handle configuration in your init() function. -for example cmd/root.go: +For example cmd/root.go: ```go +import ( + "fmt" + "os" + + homedir "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + func init() { - cobra.OnInitialize(initConfig) - RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cobra.yaml)") - RootCmd.PersistentFlags().StringVarP(&projectBase, "projectbase", "b", "", "base project directory eg. github.com/spf13/") - RootCmd.PersistentFlags().StringP("author", "a", "YOUR NAME", "Author name for copyright attribution") - RootCmd.PersistentFlags().StringVarP(&userLicense, "license", "l", "", "Name of license for the project (can provide `licensetext` in config)") - RootCmd.PersistentFlags().Bool("viper", true, "Use Viper for configuration") - viper.BindPFlag("author", RootCmd.PersistentFlags().Lookup("author")) - viper.BindPFlag("projectbase", RootCmd.PersistentFlags().Lookup("projectbase")) - viper.BindPFlag("useViper", RootCmd.PersistentFlags().Lookup("viper")) - viper.SetDefault("author", "NAME HERE ") - viper.SetDefault("license", "apache") + cobra.OnInitialize(initConfig) + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cobra.yaml)") + rootCmd.PersistentFlags().StringVarP(&projectBase, "projectbase", "b", "", "base project directory eg. github.com/spf13/") + rootCmd.PersistentFlags().StringP("author", "a", "YOUR NAME", "Author name for copyright attribution") + rootCmd.PersistentFlags().StringVarP(&userLicense, "license", "l", "", "Name of license for the project (can provide `licensetext` in config)") + rootCmd.PersistentFlags().Bool("viper", true, "Use Viper for configuration") + viper.BindPFlag("author", rootCmd.PersistentFlags().Lookup("author")) + viper.BindPFlag("projectbase", rootCmd.PersistentFlags().Lookup("projectbase")) + viper.BindPFlag("useViper", rootCmd.PersistentFlags().Lookup("viper")) + viper.SetDefault("author", "NAME HERE ") + viper.SetDefault("license", "apache") +} + +func initConfig() { + // Don't forget to read config either from cfgFile or from home directory! + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Find home directory. + home, err := homedir.Dir() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + // Search config in home directory with name ".cobra" (without extension). + viper.AddConfigPath(home) + viper.SetConfigName(".cobra") + } + + if err := viper.ReadInConfig(); err != nil { + fmt.Println("Can't read config:", err) + os.Exit(1) + } } ``` @@ -320,21 +265,17 @@ In a Cobra app, typically the main.go file is very bare. It serves, one purpose, package main import ( - "fmt" - "os" + "fmt" + "os" - "{pathToYourApp}/cmd" + "{pathToYourApp}/cmd" ) func main() { - if err := cmd.RootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } + cmd.Execute() } ``` - ### Create additional commands Additional commands can be defined and typically are each given their own file @@ -347,48 +288,25 @@ populate it with the following: package cmd import ( - "github.com/spf13/cobra" - "fmt" + "fmt" + + "github.com/spf13/cobra" ) func init() { - RootCmd.AddCommand(versionCmd) + rootCmd.AddCommand(versionCmd) } var versionCmd = &cobra.Command{ - Use: "version", - Short: "Print the version number of Hugo", - Long: `All software has versions. This is Hugo's`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("Hugo Static Site Generator v0.9 -- HEAD") - }, + Use: "version", + Short: "Print the version number of Hugo", + Long: `All software has versions. This is Hugo's`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Hugo Static Site Generator v0.9 -- HEAD") + }, } ``` -### Attach command to its parent - - -If you notice in the above example we attach the command to its parent. In -this case the parent is the rootCmd. In this example we are attaching it to the -root, but commands can be attached at any level. - -```go -RootCmd.AddCommand(versionCmd) -``` - -### Remove a command from its parent - -Removing a command is not a common action in simple programs, but it allows 3rd -parties to customize an existing command tree. - -In this example, we remove the existing `VersionCmd` command of an existing -root command, and we replace it with our own version: - -```go -mainlib.RootCmd.RemoveCommand(mainlib.VersionCmd) -mainlib.RootCmd.AddCommand(versionCmd) -``` - ## Working with Flags Flags provide modifiers to control how the action command operates. @@ -413,7 +331,7 @@ command it's assigned to as well as every command under that command. For global flags, assign a flag as a persistent flag on the root. ```go -RootCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output") +rootCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output") ``` ### Local Flags @@ -421,9 +339,83 @@ RootCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose out A flag can also be assigned locally which will only apply to that specific command. ```go -RootCmd.Flags().StringVarP(&Source, "source", "s", "", "Source directory to read from") +rootCmd.Flags().StringVarP(&Source, "source", "s", "", "Source directory to read from") ``` +### Local Flag on Parent Commands + +By default Cobra only parses local flags on the target command, any local flags on +parent commands are ignored. By enabling `Command.TraverseChildren` Cobra will +parse local flags on each command before executing the target command. + +```go +command := cobra.Command{ + Use: "print [OPTIONS] [COMMANDS]", + TraverseChildren: true, +} +``` + +### Bind Flags with Config + +You can also bind your flags with [viper](https://github.com/spf13/viper): +```go +var author string + +func init() { + rootCmd.PersistentFlags().StringVar(&author, "author", "YOUR NAME", "Author name for copyright attribution") + viper.BindPFlag("author", rootCmd.PersistentFlags().Lookup("author")) +} +``` + +In this example the persistent flag `author` is bound with `viper`. +**Note**, that the variable `author` will not be set to the value from config, +when the `--author` flag is not provided by user. + +More in [viper documentation](https://github.com/spf13/viper#working-with-flags). + +### Required flags + +Flags are optional by default. If instead you wish your command to report an error +when a flag has not been set, mark it as required: +```go +rootCmd.Flags().StringVarP(&Region, "region", "r", "", "AWS region (required)") +rootCmd.MarkFlagRequired("region") +``` + +## Positional and Custom Arguments + +Validation of positional arguments can be specified using the `Args` field +of `Command`. + +The following validators are built in: + +- `NoArgs` - the command will report an error if there are any positional args. +- `ArbitraryArgs` - the command will accept any args. +- `OnlyValidArgs` - the command will report an error if there are any positional args that are not in the `ValidArgs` field of `Command`. +- `MinimumNArgs(int)` - the command will report an error if there are not at least N positional args. +- `MaximumNArgs(int)` - the command will report an error if there are more than N positional args. +- `ExactArgs(int)` - the command will report an error if there are not exactly N positional args. +- `RangeArgs(min, max)` - the command will report an error if the number of args is not between the minimum and maximum number of expected args. + +An example of setting the custom validator: + +```go +var cmd = &cobra.Command{ + Short: "hello", + Args: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return errors.New("requires at least one arg") + } + if myapp.IsValidColor(args[0]) { + return nil + } + return fmt.Errorf("invalid color specified: %s", args[0]) + }, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Hello, World!") + }, +} +``` ## Example @@ -440,62 +432,62 @@ More documentation about flags is available at https://github.com/spf13/pflag package main import ( - "fmt" - "strings" + "fmt" + "strings" - "github.com/spf13/cobra" + "github.com/spf13/cobra" ) func main() { + var echoTimes int - var echoTimes int + var cmdPrint = &cobra.Command{ + Use: "print [string to print]", + Short: "Print anything to the screen", + Long: `print is for printing anything back to the screen. +For many years people have printed back to the screen.`, + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Print: " + strings.Join(args, " ")) + }, + } - var cmdPrint = &cobra.Command{ - Use: "print [string to print]", - Short: "Print anything to the screen", - Long: `print is for printing anything back to the screen. - For many years people have printed back to the screen. - `, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("Print: " + strings.Join(args, " ")) - }, - } + var cmdEcho = &cobra.Command{ + Use: "echo [string to echo]", + Short: "Echo anything to the screen", + Long: `echo is for echoing anything back. +Echo works a lot like print, except it has a child command.`, + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Print: " + strings.Join(args, " ")) + }, + } - var cmdEcho = &cobra.Command{ - Use: "echo [string to echo]", - Short: "Echo anything to the screen", - Long: `echo is for echoing anything back. - Echo works a lot like print, except it has a child command. - `, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("Print: " + strings.Join(args, " ")) - }, - } + var cmdTimes = &cobra.Command{ + Use: "times [# times] [string to echo]", + Short: "Echo anything to the screen more times", + Long: `echo things multiple times back to the user by providing +a count and a string.`, + Args: cobra.MinimumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + for i := 0; i < echoTimes; i++ { + fmt.Println("Echo: " + strings.Join(args, " ")) + } + }, + } - var cmdTimes = &cobra.Command{ - Use: "times [# times] [string to echo]", - Short: "Echo anything to the screen more times", - Long: `echo things multiple times back to the user by providing - a count and a string.`, - Run: func(cmd *cobra.Command, args []string) { - for i := 0; i < echoTimes; i++ { - fmt.Println("Echo: " + strings.Join(args, " ")) - } - }, - } + cmdTimes.Flags().IntVarP(&echoTimes, "times", "t", 1, "times to echo the input") - cmdTimes.Flags().IntVarP(&echoTimes, "times", "t", 1, "times to echo the input") - - var rootCmd = &cobra.Command{Use: "app"} - rootCmd.AddCommand(cmdPrint, cmdEcho) - cmdEcho.AddCommand(cmdTimes) - rootCmd.Execute() + var rootCmd = &cobra.Command{Use: "app"} + rootCmd.AddCommand(cmdPrint, cmdEcho) + cmdEcho.AddCommand(cmdTimes) + rootCmd.Execute() } ``` For a more complete example of a larger application, please checkout [Hugo](http://gohugo.io/). -## The Help Command +## Help Command Cobra automatically adds a help command to your application when you have subcommands. This will be called when a user runs 'app help'. Additionally, help will also @@ -508,60 +500,28 @@ create' is called. Every command will automatically have the '--help' flag adde The following output is automatically generated by Cobra. Nothing beyond the command and flag definitions are needed. - > hugo help + $ cobra help - hugo is the main command, used to build your Hugo site. - - Hugo is a Fast and Flexible Static Site Generator - built with love by spf13 and friends in Go. - - Complete documentation is available at http://gohugo.io/. + Cobra is a CLI library for Go that empowers applications. + This application is a tool to generate the needed files + to quickly create a Cobra application. Usage: - hugo [flags] - hugo [command] + cobra [command] Available Commands: - server Hugo runs its own webserver to render the files - version Print the version number of Hugo - config Print the site configuration - check Check content in the source directory - benchmark Benchmark hugo by building a site a number of times. - convert Convert your content to different formats - new Create new content for your site - list Listing out various types of content - undraft Undraft changes the content's draft status from 'True' to 'False' - genautocomplete Generate shell autocompletion script for Hugo - gendoc Generate Markdown documentation for the Hugo CLI. - genman Generate man page for Hugo - import Import your site from others. + add Add a command to a Cobra Application + help Help about any command + init Initialize a Cobra Application Flags: - -b, --baseURL="": hostname (and path) to the root, e.g. http://spf13.com/ - -D, --buildDrafts[=false]: include content marked as draft - -F, --buildFuture[=false]: include content with publishdate in the future - --cacheDir="": filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/ - --canonifyURLs[=false]: if true, all relative URLs will be canonicalized using baseURL - --config="": config file (default is path/config.yaml|json|toml) - -d, --destination="": filesystem path to write files to - --disableRSS[=false]: Do not build RSS files - --disableSitemap[=false]: Do not build Sitemap file - --editor="": edit new content with this editor, if provided - --ignoreCache[=false]: Ignores the cache directory for reading but still writes to it - --log[=false]: Enable Logging - --logFile="": Log File path (if set, logging enabled automatically) - --noTimes[=false]: Don't sync modification time of files - --pluralizeListTitles[=true]: Pluralize titles in lists using inflect - --preserveTaxonomyNames[=false]: Preserve taxonomy names as written ("Gérard Depardieu" vs "gerard-depardieu") - -s, --source="": filesystem path to read files relative from - --stepAnalysis[=false]: display memory and timing of different steps of the program - -t, --theme="": theme to use (located in /themes/THEMENAME/) - --uglyURLs[=false]: if true, use /filename.html instead of /filename/ - -v, --verbose[=false]: verbose output - --verboseLog[=false]: verbose logging - -w, --watch[=false]: watch filesystem for changes and recreate as needed + -a, --author string author name for copyright attribution (default "YOUR NAME") + --config string config file (default is $HOME/.cobra.yaml) + -h, --help help for cobra + -l, --license string name of license for the project + --viper use Viper for configuration (default true) - Use "hugo [command] --help" for more information about a command. + Use "cobra [command] --help" for more information about a command. Help is just a command like any other. There is no special logic or behavior @@ -569,38 +529,18 @@ around it. In fact, you can provide your own if you want. ### Defining your own help -You can provide your own Help command or your own template for the default command to use. - -The default help command is +You can provide your own Help command or your own template for the default command to use +with following functions: ```go -func (c *Command) initHelp() { - if c.helpCommand == nil { - c.helpCommand = &Command{ - Use: "help [command]", - Short: "Help about any command", - Long: `Help provides help for any command in the application. - Simply type ` + c.Name() + ` help [path to command] for full details.`, - Run: c.HelpFunc(), - } - } - c.AddCommand(c.helpCommand) -} -``` - -You can provide your own command, function or template through the following methods: - -```go -command.SetHelpCommand(cmd *Command) - -command.SetHelpFunc(f func(*Command, []string)) - -command.SetHelpTemplate(s string) +cmd.SetHelpCommand(cmd *Command) +cmd.SetHelpFunc(f func(*Command, []string)) +cmd.SetHelpTemplate(s string) ``` The latter two will also apply to any children commands. -## Usage +## Usage Message When the user provides an invalid flag or invalid command, Cobra responds by showing the user the 'usage'. @@ -609,71 +549,42 @@ showing the user the 'usage'. You may recognize this from the help above. That's because the default help embeds the usage as part of its output. + $ cobra --invalid + Error: unknown flag: --invalid Usage: - hugo [flags] - hugo [command] + cobra [command] Available Commands: - server Hugo runs its own webserver to render the files - version Print the version number of Hugo - config Print the site configuration - check Check content in the source directory - benchmark Benchmark hugo by building a site a number of times. - convert Convert your content to different formats - new Create new content for your site - list Listing out various types of content - undraft Undraft changes the content's draft status from 'True' to 'False' - genautocomplete Generate shell autocompletion script for Hugo - gendoc Generate Markdown documentation for the Hugo CLI. - genman Generate man page for Hugo - import Import your site from others. + add Add a command to a Cobra Application + help Help about any command + init Initialize a Cobra Application Flags: - -b, --baseURL="": hostname (and path) to the root, e.g. http://spf13.com/ - -D, --buildDrafts[=false]: include content marked as draft - -F, --buildFuture[=false]: include content with publishdate in the future - --cacheDir="": filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/ - --canonifyURLs[=false]: if true, all relative URLs will be canonicalized using baseURL - --config="": config file (default is path/config.yaml|json|toml) - -d, --destination="": filesystem path to write files to - --disableRSS[=false]: Do not build RSS files - --disableSitemap[=false]: Do not build Sitemap file - --editor="": edit new content with this editor, if provided - --ignoreCache[=false]: Ignores the cache directory for reading but still writes to it - --log[=false]: Enable Logging - --logFile="": Log File path (if set, logging enabled automatically) - --noTimes[=false]: Don't sync modification time of files - --pluralizeListTitles[=true]: Pluralize titles in lists using inflect - --preserveTaxonomyNames[=false]: Preserve taxonomy names as written ("Gérard Depardieu" vs "gerard-depardieu") - -s, --source="": filesystem path to read files relative from - --stepAnalysis[=false]: display memory and timing of different steps of the program - -t, --theme="": theme to use (located in /themes/THEMENAME/) - --uglyURLs[=false]: if true, use /filename.html instead of /filename/ - -v, --verbose[=false]: verbose output - --verboseLog[=false]: verbose logging - -w, --watch[=false]: watch filesystem for changes and recreate as needed + -a, --author string author name for copyright attribution (default "YOUR NAME") + --config string config file (default is $HOME/.cobra.yaml) + -h, --help help for cobra + -l, --license string name of license for the project + --viper use Viper for configuration (default true) + + Use "cobra [command] --help" for more information about a command. ### Defining your own usage You can provide your own usage function or template for Cobra to use. - -The default usage function is: - -```go -return func(c *Command) error { - err := tmpl(c.Out(), c.UsageTemplate(), c) - return err -} -``` - Like help, the function and template are overridable through public methods: ```go -command.SetUsageFunc(f func(*Command) error) - -command.SetUsageTemplate(s string) +cmd.SetUsageFunc(f func(*Command) error) +cmd.SetUsageTemplate(s string) ``` -## PreRun or PostRun Hooks +## Version Flag + +Cobra adds a top-level '--version' flag if the Version field is set on the root command. +Running an application with the '--version' flag will print the version to stdout using +the version template. The template can be customized using the +`cmd.SetVersionTemplate(s string)` function. + +## PreRun and PostRun Hooks It is possible to run functions before or after the main `Run` function of your command. The `PersistentPreRun` and `PreRun` functions will be executed before `Run`. `PersistentPostRun` and `PostRun` will be executed after `Run`. The `Persistent*Run` functions will be inherited by children if they do not declare their own. These functions are run in the following order: @@ -689,105 +600,73 @@ An example of two commands which use all of these features is below. When the s package main import ( - "fmt" + "fmt" - "github.com/spf13/cobra" + "github.com/spf13/cobra" ) func main() { - var rootCmd = &cobra.Command{ - Use: "root [sub]", - Short: "My root command", - PersistentPreRun: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside rootCmd PersistentPreRun with args: %v\n", args) - }, - PreRun: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside rootCmd PreRun with args: %v\n", args) - }, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside rootCmd Run with args: %v\n", args) - }, - PostRun: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside rootCmd PostRun with args: %v\n", args) - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside rootCmd PersistentPostRun with args: %v\n", args) - }, - } + var rootCmd = &cobra.Command{ + Use: "root [sub]", + Short: "My root command", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside rootCmd PersistentPreRun with args: %v\n", args) + }, + PreRun: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside rootCmd PreRun with args: %v\n", args) + }, + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside rootCmd Run with args: %v\n", args) + }, + PostRun: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside rootCmd PostRun with args: %v\n", args) + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside rootCmd PersistentPostRun with args: %v\n", args) + }, + } - var subCmd = &cobra.Command{ - Use: "sub [no options!]", - Short: "My subcommand", - PreRun: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside subCmd PreRun with args: %v\n", args) - }, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside subCmd Run with args: %v\n", args) - }, - PostRun: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside subCmd PostRun with args: %v\n", args) - }, - PersistentPostRun: func(cmd *cobra.Command, args []string) { - fmt.Printf("Inside subCmd PersistentPostRun with args: %v\n", args) - }, - } + var subCmd = &cobra.Command{ + Use: "sub [no options!]", + Short: "My subcommand", + PreRun: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside subCmd PreRun with args: %v\n", args) + }, + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside subCmd Run with args: %v\n", args) + }, + PostRun: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside subCmd PostRun with args: %v\n", args) + }, + PersistentPostRun: func(cmd *cobra.Command, args []string) { + fmt.Printf("Inside subCmd PersistentPostRun with args: %v\n", args) + }, + } - rootCmd.AddCommand(subCmd) + rootCmd.AddCommand(subCmd) - rootCmd.SetArgs([]string{""}) - _ = rootCmd.Execute() - fmt.Print("\n") - rootCmd.SetArgs([]string{"sub", "arg1", "arg2"}) - _ = rootCmd.Execute() + rootCmd.SetArgs([]string{""}) + rootCmd.Execute() + fmt.Println() + rootCmd.SetArgs([]string{"sub", "arg1", "arg2"}) + rootCmd.Execute() } ``` +Output: +``` +Inside rootCmd PersistentPreRun with args: [] +Inside rootCmd PreRun with args: [] +Inside rootCmd Run with args: [] +Inside rootCmd PostRun with args: [] +Inside rootCmd PersistentPostRun with args: [] -## Alternative Error Handling - -Cobra also has functions where the return signature is an error. This allows for errors to bubble up to the top, -providing a way to handle the errors in one location. The current list of functions that return an error is: - -* PersistentPreRunE -* PreRunE -* RunE -* PostRunE -* PersistentPostRunE - -If you would like to silence the default `error` and `usage` output in favor of your own, you can set `SilenceUsage` -and `SilenceErrors` to `true` on the command. A child command respects these flags if they are set on the parent -command. - -**Example Usage using RunE:** - -```go -package main - -import ( - "errors" - "log" - - "github.com/spf13/cobra" -) - -func main() { - var rootCmd = &cobra.Command{ - Use: "hugo", - Short: "Hugo is a very fast static site generator", - Long: `A Fast and Flexible Static Site Generator built with - love by spf13 and friends in Go. - Complete documentation is available at http://hugo.spf13.com`, - RunE: func(cmd *cobra.Command, args []string) error { - // Do Stuff Here - return errors.New("some random error") - }, - } - - if err := rootCmd.Execute(); err != nil { - log.Fatal(err) - } -} +Inside rootCmd PersistentPreRun with args: [arg1 arg2] +Inside subCmd PreRun with args: [arg1 arg2] +Inside subCmd Run with args: [arg1 arg2] +Inside subCmd PostRun with args: [arg1 arg2] +Inside subCmd PersistentPostRun with args: [arg1 arg2] ``` ## Suggestions when "unknown command" happens @@ -830,81 +709,28 @@ Did you mean this? Run 'kubectl help' for usage. ``` -## Generating Markdown-formatted documentation for your command +## Generating documentation for your command -Cobra can generate a Markdown-formatted document based on the subcommands, flags, etc. A simple example of how to do this for your command can be found in [Markdown Docs](doc/md_docs.md). +Cobra can generate documentation based on subcommands, flags, etc. in the following formats: -## Generating man pages for your command +- [Markdown](doc/md_docs.md) +- [ReStructured Text](doc/rest_docs.md) +- [Man Page](doc/man_docs.md) -Cobra can generate a man page based on the subcommands, flags, etc. A simple example of how to do this for your command can be found in [Man Docs](doc/man_docs.md). - -## Generating bash completions for your command +## Generating bash completions Cobra can generate a bash-completion file. If you add more information to your command, these completions can be amazingly powerful and flexible. Read more about it in [Bash Completions](bash_completions.md). -## Debugging - -Cobra provides a ‘DebugFlags’ method on a command which, when called, will print -out everything Cobra knows about the flags for each command. - -### Example - -```go -command.DebugFlags() -``` - -## Release Notes -* **0.9.0** June 17, 2014 - * flags can appears anywhere in the args (provided they are unambiguous) - * --help prints usage screen for app or command - * Prefix matching for commands - * Cleaner looking help and usage output - * Extensive test suite -* **0.8.0** Nov 5, 2013 - * Reworked interface to remove commander completely - * Command now primary structure - * No initialization needed - * Usage & Help templates & functions definable at any level - * Updated Readme -* **0.7.0** Sept 24, 2013 - * Needs more eyes - * Test suite - * Support for automatic error messages - * Support for help command - * Support for printing to any io.Writer instead of os.Stderr - * Support for persistent flags which cascade down tree - * Ready for integration into Hugo -* **0.1.0** Sept 3, 2013 - * Implement first draft - -## Extensions - -Libraries for extending Cobra: - -* [cmdns](https://github.com/gosuri/cmdns): Enables name spacing a command's immediate children. It provides an alternative way to structure subcommands, similar to `heroku apps:create` and `ovrclk clusters:launch`. - -## ToDo -* Launch proper documentation site - -## Contributing +# Contributing 1. Fork it -2. Create your feature branch (`git checkout -b my-new-feature`) -3. Commit your changes (`git commit -am 'Add some feature'`) -4. Push to the branch (`git push origin my-new-feature`) -5. Create new Pull Request +2. Download your fork to your PC (`git clone https://github.com/your_username/cobra && cd cobra`) +3. Create your feature branch (`git checkout -b my-new-feature`) +4. Make changes and add them (`git add .`) +5. Commit your changes (`git commit -m 'Add some feature'`) +6. Push to the branch (`git push origin my-new-feature`) +7. Create new pull request -## Contributors - -Names in no particular order: - -* [spf13](https://github.com/spf13), -[eparis](https://github.com/eparis), -[bep](https://github.com/bep), and many more! - -## License +# License Cobra is released under the Apache 2.0 license. See [LICENSE.txt](https://github.com/spf13/cobra/blob/master/LICENSE.txt) - - -[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/spf13/cobra/trend.png)](https://bitdeli.com/free "Bitdeli Badge") diff --git a/vendor/github.com/spf13/cobra/args.go b/vendor/github.com/spf13/cobra/args.go new file mode 100644 index 00000000..a5d8a927 --- /dev/null +++ b/vendor/github.com/spf13/cobra/args.go @@ -0,0 +1,89 @@ +package cobra + +import ( + "fmt" +) + +type PositionalArgs func(cmd *Command, args []string) error + +// Legacy arg validation has the following behaviour: +// - root commands with no subcommands can take arbitrary arguments +// - root commands with subcommands will do subcommand validity checking +// - subcommands will always accept arbitrary arguments +func legacyArgs(cmd *Command, args []string) error { + // no subcommand, always take args + if !cmd.HasSubCommands() { + return nil + } + + // root command with subcommands, do subcommand checking. + if !cmd.HasParent() && len(args) > 0 { + return fmt.Errorf("unknown command %q for %q%s", args[0], cmd.CommandPath(), cmd.findSuggestions(args[0])) + } + return nil +} + +// NoArgs returns an error if any args are included. +func NoArgs(cmd *Command, args []string) error { + if len(args) > 0 { + return fmt.Errorf("unknown command %q for %q", args[0], cmd.CommandPath()) + } + return nil +} + +// OnlyValidArgs returns an error if any args are not in the list of ValidArgs. +func OnlyValidArgs(cmd *Command, args []string) error { + if len(cmd.ValidArgs) > 0 { + for _, v := range args { + if !stringInSlice(v, cmd.ValidArgs) { + return fmt.Errorf("invalid argument %q for %q%s", v, cmd.CommandPath(), cmd.findSuggestions(args[0])) + } + } + } + return nil +} + +// ArbitraryArgs never returns an error. +func ArbitraryArgs(cmd *Command, args []string) error { + return nil +} + +// MinimumNArgs returns an error if there is not at least N args. +func MinimumNArgs(n int) PositionalArgs { + return func(cmd *Command, args []string) error { + if len(args) < n { + return fmt.Errorf("requires at least %d arg(s), only received %d", n, len(args)) + } + return nil + } +} + +// MaximumNArgs returns an error if there are more than N args. +func MaximumNArgs(n int) PositionalArgs { + return func(cmd *Command, args []string) error { + if len(args) > n { + return fmt.Errorf("accepts at most %d arg(s), received %d", n, len(args)) + } + return nil + } +} + +// ExactArgs returns an error if there are not exactly n args. +func ExactArgs(n int) PositionalArgs { + return func(cmd *Command, args []string) error { + if len(args) != n { + return fmt.Errorf("accepts %d arg(s), received %d", n, len(args)) + } + return nil + } +} + +// RangeArgs returns an error if the number of args is not within the expected range. +func RangeArgs(min int, max int) PositionalArgs { + return func(cmd *Command, args []string) error { + if len(args) < min || len(args) > max { + return fmt.Errorf("accepts between %d and %d arg(s), received %d", min, max, len(args)) + } + return nil + } +} diff --git a/vendor/github.com/spf13/cobra/bash_completions.go b/vendor/github.com/spf13/cobra/bash_completions.go index a0d04027..8fa8f486 100644 --- a/vendor/github.com/spf13/cobra/bash_completions.go +++ b/vendor/github.com/spf13/cobra/bash_completions.go @@ -1,6 +1,7 @@ package cobra import ( + "bytes" "fmt" "io" "os" @@ -18,13 +19,10 @@ const ( BashCompSubdirsInDir = "cobra_annotation_bash_completion_subdirs_in_dir" ) -func preamble(out io.Writer, name string) error { - _, err := fmt.Fprintf(out, "# bash completion for %-36s -*- shell-script -*-\n", name) - if err != nil { - return err - } - preamStr := ` -__debug() +func writePreamble(buf *bytes.Buffer, name string) { + buf.WriteString(fmt.Sprintf("# bash completion for %-36s -*- shell-script -*-\n", name)) + buf.WriteString(fmt.Sprintf(` +__%[1]s_debug() { if [[ -n ${BASH_COMP_DEBUG_FILE} ]]; then echo "$*" >> "${BASH_COMP_DEBUG_FILE}" @@ -33,13 +31,13 @@ __debug() # Homebrew on Macs have version 1.3 of bash-completion which doesn't include # _init_completion. This is a very minimal version of that function. -__my_init_completion() +__%[1]s_init_completion() { COMPREPLY=() _get_comp_words_by_ref "$@" cur prev words cword } -__index_of_word() +__%[1]s_index_of_word() { local w word=$1 shift @@ -51,7 +49,7 @@ __index_of_word() index=-1 } -__contains_word() +__%[1]s_contains_word() { local w word=$1; shift for w in "$@"; do @@ -60,9 +58,9 @@ __contains_word() return 1 } -__handle_reply() +__%[1]s_handle_reply() { - __debug "${FUNCNAME[0]}" + __%[1]s_debug "${FUNCNAME[0]}" case $cur in -*) if [[ $(type -t compopt) = "builtin" ]]; then @@ -87,14 +85,14 @@ __handle_reply() local index flag flag="${cur%%=*}" - __index_of_word "${flag}" "${flags_with_completion[@]}" + __%[1]s_index_of_word "${flag}" "${flags_with_completion[@]}" COMPREPLY=() if [[ ${index} -ge 0 ]]; then PREFIX="" cur="${cur#*=}" ${flags_completion[${index}]} if [ -n "${ZSH_VERSION}" ]; then - # zfs completion needs --flag= prefix + # zsh completion needs --flag= prefix eval "COMPREPLY=( \"\${COMPREPLY[@]/#/${flag}=}\" )" fi fi @@ -105,7 +103,7 @@ __handle_reply() # check if we are handling a flag with special work handling local index - __index_of_word "${prev}" "${flags_with_completion[@]}" + __%[1]s_index_of_word "${prev}" "${flags_with_completion[@]}" if [[ ${index} -ge 0 ]]; then ${flags_completion[${index}]} return @@ -134,25 +132,34 @@ __handle_reply() declare -F __custom_func >/dev/null && __custom_func fi - __ltrim_colon_completions "$cur" + # available in bash-completion >= 2, not always present on macOS + if declare -F __ltrim_colon_completions >/dev/null; then + __ltrim_colon_completions "$cur" + fi + + # If there is only 1 completion and it is a flag with an = it will be completed + # but we don't want a space after the = + if [[ "${#COMPREPLY[@]}" -eq "1" ]] && [[ $(type -t compopt) = "builtin" ]] && [[ "${COMPREPLY[0]}" == --*= ]]; then + compopt -o nospace + fi } # The arguments should be in the form "ext1|ext2|extn" -__handle_filename_extension_flag() +__%[1]s_handle_filename_extension_flag() { local ext="$1" _filedir "@(${ext})" } -__handle_subdirs_in_dir_flag() +__%[1]s_handle_subdirs_in_dir_flag() { local dir="$1" pushd "${dir}" >/dev/null 2>&1 && _filedir -d && popd >/dev/null 2>&1 } -__handle_flag() +__%[1]s_handle_flag() { - __debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" + __%[1]s_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" # if a command required a flag, and we found it, unset must_have_one_flag() local flagname=${words[c]} @@ -163,27 +170,30 @@ __handle_flag() flagname=${flagname%%=*} # strip everything after the = flagname="${flagname}=" # but put the = back fi - __debug "${FUNCNAME[0]}: looking for ${flagname}" - if __contains_word "${flagname}" "${must_have_one_flag[@]}"; then + __%[1]s_debug "${FUNCNAME[0]}: looking for ${flagname}" + if __%[1]s_contains_word "${flagname}" "${must_have_one_flag[@]}"; then must_have_one_flag=() fi # if you set a flag which only applies to this command, don't show subcommands - if __contains_word "${flagname}" "${local_nonpersistent_flags[@]}"; then + if __%[1]s_contains_word "${flagname}" "${local_nonpersistent_flags[@]}"; then commands=() fi # keep flag value with flagname as flaghash - if [ -n "${flagvalue}" ] ; then - flaghash[${flagname}]=${flagvalue} - elif [ -n "${words[ $((c+1)) ]}" ] ; then - flaghash[${flagname}]=${words[ $((c+1)) ]} - else - flaghash[${flagname}]="true" # pad "true" for bool flag + # flaghash variable is an associative array which is only supported in bash > 3. + if [[ -z "${BASH_VERSION}" || "${BASH_VERSINFO[0]}" -gt 3 ]]; then + if [ -n "${flagvalue}" ] ; then + flaghash[${flagname}]=${flagvalue} + elif [ -n "${words[ $((c+1)) ]}" ] ; then + flaghash[${flagname}]=${words[ $((c+1)) ]} + else + flaghash[${flagname}]="true" # pad "true" for bool flag + fi fi # skip the argument to a two word flag - if __contains_word "${words[c]}" "${two_word_flags[@]}"; then + if __%[1]s_contains_word "${words[c]}" "${two_word_flags[@]}"; then c=$((c+1)) # if we are looking for a flags value, don't show commands if [[ $c -eq $cword ]]; then @@ -195,13 +205,13 @@ __handle_flag() } -__handle_noun() +__%[1]s_handle_noun() { - __debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" + __%[1]s_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" - if __contains_word "${words[c]}" "${must_have_one_noun[@]}"; then + if __%[1]s_contains_word "${words[c]}" "${must_have_one_noun[@]}"; then must_have_one_noun=() - elif __contains_word "${words[c]}" "${noun_aliases[@]}"; then + elif __%[1]s_contains_word "${words[c]}" "${noun_aliases[@]}"; then must_have_one_noun=() fi @@ -209,62 +219,66 @@ __handle_noun() c=$((c+1)) } -__handle_command() +__%[1]s_handle_command() { - __debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" + __%[1]s_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" local next_command if [[ -n ${last_command} ]]; then next_command="_${last_command}_${words[c]//:/__}" else if [[ $c -eq 0 ]]; then - next_command="_$(basename "${words[c]//:/__}")" + next_command="_%[1]s_root_command" else next_command="_${words[c]//:/__}" fi fi c=$((c+1)) - __debug "${FUNCNAME[0]}: looking for ${next_command}" + __%[1]s_debug "${FUNCNAME[0]}: looking for ${next_command}" declare -F "$next_command" >/dev/null && $next_command } -__handle_word() +__%[1]s_handle_word() { if [[ $c -ge $cword ]]; then - __handle_reply + __%[1]s_handle_reply return fi - __debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" + __%[1]s_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}" if [[ "${words[c]}" == -* ]]; then - __handle_flag - elif __contains_word "${words[c]}" "${commands[@]}"; then - __handle_command - elif [[ $c -eq 0 ]] && __contains_word "$(basename "${words[c]}")" "${commands[@]}"; then - __handle_command + __%[1]s_handle_flag + elif __%[1]s_contains_word "${words[c]}" "${commands[@]}"; then + __%[1]s_handle_command + elif [[ $c -eq 0 ]]; then + __%[1]s_handle_command + elif __%[1]s_contains_word "${words[c]}" "${command_aliases[@]}"; then + # aliashash variable is an associative array which is only supported in bash > 3. + if [[ -z "${BASH_VERSION}" || "${BASH_VERSINFO[0]}" -gt 3 ]]; then + words[c]=${aliashash[${words[c]}]} + __%[1]s_handle_command + else + __%[1]s_handle_noun + fi else - __handle_noun + __%[1]s_handle_noun fi - __handle_word + __%[1]s_handle_word } -` - _, err = fmt.Fprint(out, preamStr) - return err +`, name)) } -func postscript(w io.Writer, name string) error { +func writePostscript(buf *bytes.Buffer, name string) { name = strings.Replace(name, ":", "__", -1) - _, err := fmt.Fprintf(w, "__start_%s()\n", name) - if err != nil { - return err - } - _, err = fmt.Fprintf(w, `{ + buf.WriteString(fmt.Sprintf("__start_%s()\n", name)) + buf.WriteString(fmt.Sprintf(`{ local cur prev words cword declare -A flaghash 2>/dev/null || : + declare -A aliashash 2>/dev/null || : if declare -F _init_completion >/dev/null 2>&1; then _init_completion -s || return else - __my_init_completion -n "=" || return + __%[1]s_init_completion -n "=" || return fi local c=0 @@ -273,206 +287,142 @@ func postscript(w io.Writer, name string) error { local local_nonpersistent_flags=() local flags_with_completion=() local flags_completion=() - local commands=("%s") + local commands=("%[1]s") local must_have_one_flag=() local must_have_one_noun=() local last_command local nouns=() - __handle_word + __%[1]s_handle_word } -`, name) - if err != nil { - return err - } - _, err = fmt.Fprintf(w, `if [[ $(type -t compopt) = "builtin" ]]; then +`, name)) + buf.WriteString(fmt.Sprintf(`if [[ $(type -t compopt) = "builtin" ]]; then complete -o default -F __start_%s %s else complete -o default -o nospace -F __start_%s %s fi -`, name, name, name, name) - if err != nil { - return err - } - _, err = fmt.Fprintf(w, "# ex: ts=4 sw=4 et filetype=sh\n") - return err +`, name, name, name, name)) + buf.WriteString("# ex: ts=4 sw=4 et filetype=sh\n") } -func writeCommands(cmd *Command, w io.Writer) error { - if _, err := fmt.Fprintf(w, " commands=()\n"); err != nil { - return err - } +func writeCommands(buf *bytes.Buffer, cmd *Command) { + buf.WriteString(" commands=()\n") for _, c := range cmd.Commands() { if !c.IsAvailableCommand() || c == cmd.helpCommand { continue } - if _, err := fmt.Fprintf(w, " commands+=(%q)\n", c.Name()); err != nil { - return err - } + buf.WriteString(fmt.Sprintf(" commands+=(%q)\n", c.Name())) + writeCmdAliases(buf, c) } - _, err := fmt.Fprintf(w, "\n") - return err + buf.WriteString("\n") } -func writeFlagHandler(name string, annotations map[string][]string, w io.Writer) error { +func writeFlagHandler(buf *bytes.Buffer, name string, annotations map[string][]string, cmd *Command) { for key, value := range annotations { switch key { case BashCompFilenameExt: - _, err := fmt.Fprintf(w, " flags_with_completion+=(%q)\n", name) - if err != nil { - return err - } + buf.WriteString(fmt.Sprintf(" flags_with_completion+=(%q)\n", name)) + var ext string if len(value) > 0 { - ext := "__handle_filename_extension_flag " + strings.Join(value, "|") - _, err = fmt.Fprintf(w, " flags_completion+=(%q)\n", ext) + ext = fmt.Sprintf("__%s_handle_filename_extension_flag ", cmd.Root().Name()) + strings.Join(value, "|") } else { - ext := "_filedir" - _, err = fmt.Fprintf(w, " flags_completion+=(%q)\n", ext) - } - if err != nil { - return err + ext = "_filedir" } + buf.WriteString(fmt.Sprintf(" flags_completion+=(%q)\n", ext)) case BashCompCustom: - _, err := fmt.Fprintf(w, " flags_with_completion+=(%q)\n", name) - if err != nil { - return err - } + buf.WriteString(fmt.Sprintf(" flags_with_completion+=(%q)\n", name)) if len(value) > 0 { handlers := strings.Join(value, "; ") - _, err = fmt.Fprintf(w, " flags_completion+=(%q)\n", handlers) + buf.WriteString(fmt.Sprintf(" flags_completion+=(%q)\n", handlers)) } else { - _, err = fmt.Fprintf(w, " flags_completion+=(:)\n") - } - if err != nil { - return err + buf.WriteString(" flags_completion+=(:)\n") } case BashCompSubdirsInDir: - _, err := fmt.Fprintf(w, " flags_with_completion+=(%q)\n", name) + buf.WriteString(fmt.Sprintf(" flags_with_completion+=(%q)\n", name)) + var ext string if len(value) == 1 { - ext := "__handle_subdirs_in_dir_flag " + value[0] - _, err = fmt.Fprintf(w, " flags_completion+=(%q)\n", ext) + ext = fmt.Sprintf("__%s_handle_subdirs_in_dir_flag ", cmd.Root().Name()) + value[0] } else { - ext := "_filedir -d" - _, err = fmt.Fprintf(w, " flags_completion+=(%q)\n", ext) - } - if err != nil { - return err + ext = "_filedir -d" } + buf.WriteString(fmt.Sprintf(" flags_completion+=(%q)\n", ext)) } } - return nil } -func writeShortFlag(flag *pflag.Flag, w io.Writer) error { - b := (len(flag.NoOptDefVal) > 0) +func writeShortFlag(buf *bytes.Buffer, flag *pflag.Flag, cmd *Command) { name := flag.Shorthand format := " " - if !b { + if len(flag.NoOptDefVal) == 0 { format += "two_word_" } format += "flags+=(\"-%s\")\n" - if _, err := fmt.Fprintf(w, format, name); err != nil { - return err - } - return writeFlagHandler("-"+name, flag.Annotations, w) + buf.WriteString(fmt.Sprintf(format, name)) + writeFlagHandler(buf, "-"+name, flag.Annotations, cmd) } -func writeFlag(flag *pflag.Flag, w io.Writer) error { - b := (len(flag.NoOptDefVal) > 0) +func writeFlag(buf *bytes.Buffer, flag *pflag.Flag, cmd *Command) { name := flag.Name format := " flags+=(\"--%s" - if !b { + if len(flag.NoOptDefVal) == 0 { format += "=" } format += "\")\n" - if _, err := fmt.Fprintf(w, format, name); err != nil { - return err - } - return writeFlagHandler("--"+name, flag.Annotations, w) + buf.WriteString(fmt.Sprintf(format, name)) + writeFlagHandler(buf, "--"+name, flag.Annotations, cmd) } -func writeLocalNonPersistentFlag(flag *pflag.Flag, w io.Writer) error { - b := (len(flag.NoOptDefVal) > 0) +func writeLocalNonPersistentFlag(buf *bytes.Buffer, flag *pflag.Flag) { name := flag.Name format := " local_nonpersistent_flags+=(\"--%s" - if !b { + if len(flag.NoOptDefVal) == 0 { format += "=" } format += "\")\n" - _, err := fmt.Fprintf(w, format, name) - return err + buf.WriteString(fmt.Sprintf(format, name)) } -func writeFlags(cmd *Command, w io.Writer) error { - _, err := fmt.Fprintf(w, ` flags=() +func writeFlags(buf *bytes.Buffer, cmd *Command) { + buf.WriteString(` flags=() two_word_flags=() local_nonpersistent_flags=() flags_with_completion=() flags_completion=() `) - if err != nil { - return err - } localNonPersistentFlags := cmd.LocalNonPersistentFlags() - var visitErr error cmd.NonInheritedFlags().VisitAll(func(flag *pflag.Flag) { if nonCompletableFlag(flag) { return } - if err := writeFlag(flag, w); err != nil { - visitErr = err - return - } + writeFlag(buf, flag, cmd) if len(flag.Shorthand) > 0 { - if err := writeShortFlag(flag, w); err != nil { - visitErr = err - return - } + writeShortFlag(buf, flag, cmd) } if localNonPersistentFlags.Lookup(flag.Name) != nil { - if err := writeLocalNonPersistentFlag(flag, w); err != nil { - visitErr = err - return - } + writeLocalNonPersistentFlag(buf, flag) } }) - if visitErr != nil { - return visitErr - } cmd.InheritedFlags().VisitAll(func(flag *pflag.Flag) { if nonCompletableFlag(flag) { return } - if err := writeFlag(flag, w); err != nil { - visitErr = err - return - } + writeFlag(buf, flag, cmd) if len(flag.Shorthand) > 0 { - if err := writeShortFlag(flag, w); err != nil { - visitErr = err - return - } + writeShortFlag(buf, flag, cmd) } }) - if visitErr != nil { - return visitErr - } - _, err = fmt.Fprintf(w, "\n") - return err + buf.WriteString("\n") } -func writeRequiredFlag(cmd *Command, w io.Writer) error { - if _, err := fmt.Fprintf(w, " must_have_one_flag=()\n"); err != nil { - return err - } +func writeRequiredFlag(buf *bytes.Buffer, cmd *Command) { + buf.WriteString(" must_have_one_flag=()\n") flags := cmd.NonInheritedFlags() - var visitErr error flags.VisitAll(func(flag *pflag.Flag) { if nonCompletableFlag(flag) { return @@ -481,107 +431,93 @@ func writeRequiredFlag(cmd *Command, w io.Writer) error { switch key { case BashCompOneRequiredFlag: format := " must_have_one_flag+=(\"--%s" - b := (flag.Value.Type() == "bool") - if !b { + if flag.Value.Type() != "bool" { format += "=" } format += "\")\n" - if _, err := fmt.Fprintf(w, format, flag.Name); err != nil { - visitErr = err - return - } + buf.WriteString(fmt.Sprintf(format, flag.Name)) if len(flag.Shorthand) > 0 { - if _, err := fmt.Fprintf(w, " must_have_one_flag+=(\"-%s\")\n", flag.Shorthand); err != nil { - visitErr = err - return - } + buf.WriteString(fmt.Sprintf(" must_have_one_flag+=(\"-%s\")\n", flag.Shorthand)) } } } }) - return visitErr } -func writeRequiredNouns(cmd *Command, w io.Writer) error { - if _, err := fmt.Fprintf(w, " must_have_one_noun=()\n"); err != nil { - return err - } +func writeRequiredNouns(buf *bytes.Buffer, cmd *Command) { + buf.WriteString(" must_have_one_noun=()\n") sort.Sort(sort.StringSlice(cmd.ValidArgs)) for _, value := range cmd.ValidArgs { - if _, err := fmt.Fprintf(w, " must_have_one_noun+=(%q)\n", value); err != nil { - return err - } + buf.WriteString(fmt.Sprintf(" must_have_one_noun+=(%q)\n", value)) } - return nil } -func writeArgAliases(cmd *Command, w io.Writer) error { - if _, err := fmt.Fprintf(w, " noun_aliases=()\n"); err != nil { - return err +func writeCmdAliases(buf *bytes.Buffer, cmd *Command) { + if len(cmd.Aliases) == 0 { + return } + + sort.Sort(sort.StringSlice(cmd.Aliases)) + + buf.WriteString(fmt.Sprint(` if [[ -z "${BASH_VERSION}" || "${BASH_VERSINFO[0]}" -gt 3 ]]; then`, "\n")) + for _, value := range cmd.Aliases { + buf.WriteString(fmt.Sprintf(" command_aliases+=(%q)\n", value)) + buf.WriteString(fmt.Sprintf(" aliashash[%q]=%q\n", value, cmd.Name())) + } + buf.WriteString(` fi`) + buf.WriteString("\n") +} +func writeArgAliases(buf *bytes.Buffer, cmd *Command) { + buf.WriteString(" noun_aliases=()\n") sort.Sort(sort.StringSlice(cmd.ArgAliases)) for _, value := range cmd.ArgAliases { - if _, err := fmt.Fprintf(w, " noun_aliases+=(%q)\n", value); err != nil { - return err - } + buf.WriteString(fmt.Sprintf(" noun_aliases+=(%q)\n", value)) } - return nil } -func gen(cmd *Command, w io.Writer) error { +func gen(buf *bytes.Buffer, cmd *Command) { for _, c := range cmd.Commands() { if !c.IsAvailableCommand() || c == cmd.helpCommand { continue } - if err := gen(c, w); err != nil { - return err - } + gen(buf, c) } commandName := cmd.CommandPath() commandName = strings.Replace(commandName, " ", "_", -1) commandName = strings.Replace(commandName, ":", "__", -1) - if _, err := fmt.Fprintf(w, "_%s()\n{\n", commandName); err != nil { - return err + + if cmd.Root() == cmd { + buf.WriteString(fmt.Sprintf("_%s_root_command()\n{\n", commandName)) + } else { + buf.WriteString(fmt.Sprintf("_%s()\n{\n", commandName)) } - if _, err := fmt.Fprintf(w, " last_command=%q\n", commandName); err != nil { - return err - } - if err := writeCommands(cmd, w); err != nil { - return err - } - if err := writeFlags(cmd, w); err != nil { - return err - } - if err := writeRequiredFlag(cmd, w); err != nil { - return err - } - if err := writeRequiredNouns(cmd, w); err != nil { - return err - } - if err := writeArgAliases(cmd, w); err != nil { - return err - } - if _, err := fmt.Fprintf(w, "}\n\n"); err != nil { - return err - } - return nil + + buf.WriteString(fmt.Sprintf(" last_command=%q\n", commandName)) + buf.WriteString("\n") + buf.WriteString(" command_aliases=()\n") + buf.WriteString("\n") + + writeCommands(buf, cmd) + writeFlags(buf, cmd) + writeRequiredFlag(buf, cmd) + writeRequiredNouns(buf, cmd) + writeArgAliases(buf, cmd) + buf.WriteString("}\n\n") } // GenBashCompletion generates bash completion file and writes to the passed writer. -func (cmd *Command) GenBashCompletion(w io.Writer) error { - if err := preamble(w, cmd.Name()); err != nil { - return err +func (c *Command) GenBashCompletion(w io.Writer) error { + buf := new(bytes.Buffer) + writePreamble(buf, c.Name()) + if len(c.BashCompletionFunction) > 0 { + buf.WriteString(c.BashCompletionFunction + "\n") } - if len(cmd.BashCompletionFunction) > 0 { - if _, err := fmt.Fprintf(w, "%s\n", cmd.BashCompletionFunction); err != nil { - return err - } - } - if err := gen(cmd, w); err != nil { - return err - } - return postscript(w, cmd.Name()) + gen(buf, c) + writePostscript(buf, c.Name()) + + _, err := buf.WriteTo(w) + return err } func nonCompletableFlag(flag *pflag.Flag) bool { @@ -589,47 +525,50 @@ func nonCompletableFlag(flag *pflag.Flag) bool { } // GenBashCompletionFile generates bash completion file. -func (cmd *Command) GenBashCompletionFile(filename string) error { +func (c *Command) GenBashCompletionFile(filename string) error { outFile, err := os.Create(filename) if err != nil { return err } defer outFile.Close() - return cmd.GenBashCompletion(outFile) + return c.GenBashCompletion(outFile) } -// MarkFlagRequired adds the BashCompOneRequiredFlag annotation to the named flag, if it exists. -func (cmd *Command) MarkFlagRequired(name string) error { - return MarkFlagRequired(cmd.Flags(), name) +// MarkFlagRequired adds the BashCompOneRequiredFlag annotation to the named flag if it exists, +// and causes your command to report an error if invoked without the flag. +func (c *Command) MarkFlagRequired(name string) error { + return MarkFlagRequired(c.Flags(), name) } -// MarkPersistentFlagRequired adds the BashCompOneRequiredFlag annotation to the named persistent flag, if it exists. -func (cmd *Command) MarkPersistentFlagRequired(name string) error { - return MarkFlagRequired(cmd.PersistentFlags(), name) +// MarkPersistentFlagRequired adds the BashCompOneRequiredFlag annotation to the named persistent flag if it exists, +// and causes your command to report an error if invoked without the flag. +func (c *Command) MarkPersistentFlagRequired(name string) error { + return MarkFlagRequired(c.PersistentFlags(), name) } -// MarkFlagRequired adds the BashCompOneRequiredFlag annotation to the named flag in the flag set, if it exists. +// MarkFlagRequired adds the BashCompOneRequiredFlag annotation to the named flag if it exists, +// and causes your command to report an error if invoked without the flag. func MarkFlagRequired(flags *pflag.FlagSet, name string) error { return flags.SetAnnotation(name, BashCompOneRequiredFlag, []string{"true"}) } // MarkFlagFilename adds the BashCompFilenameExt annotation to the named flag, if it exists. // Generated bash autocompletion will select filenames for the flag, limiting to named extensions if provided. -func (cmd *Command) MarkFlagFilename(name string, extensions ...string) error { - return MarkFlagFilename(cmd.Flags(), name, extensions...) +func (c *Command) MarkFlagFilename(name string, extensions ...string) error { + return MarkFlagFilename(c.Flags(), name, extensions...) } // MarkFlagCustom adds the BashCompCustom annotation to the named flag, if it exists. // Generated bash autocompletion will call the bash function f for the flag. -func (cmd *Command) MarkFlagCustom(name string, f string) error { - return MarkFlagCustom(cmd.Flags(), name, f) +func (c *Command) MarkFlagCustom(name string, f string) error { + return MarkFlagCustom(c.Flags(), name, f) } // MarkPersistentFlagFilename adds the BashCompFilenameExt annotation to the named persistent flag, if it exists. // Generated bash autocompletion will select filenames for the flag, limiting to named extensions if provided. -func (cmd *Command) MarkPersistentFlagFilename(name string, extensions ...string) error { - return MarkFlagFilename(cmd.PersistentFlags(), name, extensions...) +func (c *Command) MarkPersistentFlagFilename(name string, extensions ...string) error { + return MarkFlagFilename(c.PersistentFlags(), name, extensions...) } // MarkFlagFilename adds the BashCompFilenameExt annotation to the named flag in the flag set, if it exists. diff --git a/vendor/github.com/spf13/cobra/bash_completions.md b/vendor/github.com/spf13/cobra/bash_completions.md index 7fa970fa..e79d4769 100644 --- a/vendor/github.com/spf13/cobra/bash_completions.md +++ b/vendor/github.com/spf13/cobra/bash_completions.md @@ -6,15 +6,16 @@ Generating bash completions from a cobra command is incredibly easy. An actual p package main import ( - "io/ioutil" - "os" + "io/ioutil" + "os" - "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd" + "k8s.io/kubernetes/pkg/kubectl/cmd" + "k8s.io/kubernetes/pkg/kubectl/cmd/util" ) func main() { - kubectl := cmd.NewFactory(nil).NewKubectlCommand(os.Stdin, ioutil.Discard, ioutil.Discard) - kubectl.GenBashCompletionFile("out.sh") + kubectl := cmd.NewKubectlCommand(util.NewFactory(nil), os.Stdin, ioutil.Discard, ioutil.Discard) + kubectl.GenBashCompletionFile("out.sh") } ``` @@ -106,7 +107,7 @@ node pod replicationcontroller service If your nouns have a number of aliases, you can define them alongside `ValidArgs` using `ArgAliases`: -```go` +```go argAliases []string = { "pods", "nodes", "services", "svc", "replicationcontrollers", "rc" } cmd := &cobra.Command{ @@ -173,14 +174,14 @@ hello.yml test.json So while there are many other files in the CWD it only shows me subdirs and those with valid extensions. -# Specifiy custom flag completion +# Specify custom flag completion -Similar to the filename completion and filtering using cobra.BashCompFilenameExt, you can specifiy +Similar to the filename completion and filtering using cobra.BashCompFilenameExt, you can specify a custom flag completion function with cobra.BashCompCustom: ```go annotation := make(map[string][]string) - annotation[cobra.BashCompFilenameExt] = []string{"__kubectl_get_namespaces"} + annotation[cobra.BashCompCustom] = []string{"__kubectl_get_namespaces"} flag := &pflag.Flag{ Name: "namespace", @@ -204,3 +205,17 @@ __kubectl_get_namespaces() fi } ``` +# Using bash aliases for commands + +You can also configure the `bash aliases` for the commands and they will also support completions. + +```bash +alias aliasname=origcommand +complete -o default -F __start_origcommand aliasname + +# and now when you run `aliasname` completion will make +# suggestions as it did for `origcommand`. + +$) aliasname +completion firstcommand secondcommand +``` diff --git a/vendor/github.com/spf13/cobra/cobra.go b/vendor/github.com/spf13/cobra/cobra.go index 25473a70..7010fd15 100644 --- a/vendor/github.com/spf13/cobra/cobra.go +++ b/vendor/github.com/spf13/cobra/cobra.go @@ -27,12 +27,13 @@ import ( ) var templateFuncs = template.FuncMap{ - "trim": strings.TrimSpace, - "trimRightSpace": trimRightSpace, - "appendIfNotPresent": appendIfNotPresent, - "rpad": rpad, - "gt": Gt, - "eq": Eq, + "trim": strings.TrimSpace, + "trimRightSpace": trimRightSpace, + "trimTrailingWhitespaces": trimRightSpace, + "appendIfNotPresent": appendIfNotPresent, + "rpad": rpad, + "gt": Gt, + "eq": Eq, } var initializers []func() @@ -46,6 +47,15 @@ var EnablePrefixMatching = false // To disable sorting, set it to false. var EnableCommandSorting = true +// MousetrapHelpText enables an information splash screen on Windows +// if the CLI is started from explorer.exe. +// To disable the mousetrap, just set this variable to blank string (""). +// Works only on Microsoft Windows. +var MousetrapHelpText string = `This is a command line tool. + +You need to open cmd.exe and run it from there. +` + // AddTemplateFunc adds a template function that's available to Usage and Help // template generation. func AddTemplateFunc(name string, tmplFunc interface{}) { @@ -60,11 +70,14 @@ func AddTemplateFuncs(tmplFuncs template.FuncMap) { } } -// OnInitialize takes a series of func() arguments and appends them to a slice of func(). +// OnInitialize sets the passed functions to be run when each command's +// Execute method is called. func OnInitialize(y ...func()) { initializers = append(initializers, y...) } +// FIXME Gt is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra. + // Gt takes two types and checks whether the first type is greater than the second. In case of types Arrays, Chans, // Maps and Slices, Gt will compare their lengths. Ints are compared directly while strings are first parsed as // ints and then compared. @@ -95,6 +108,8 @@ func Gt(a interface{}, b interface{}) bool { return left > right } +// FIXME Eq is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra. + // Eq takes two types and checks whether they are equal. Supported types are int and string. Unsupported types will panic. func Eq(a interface{}, b interface{}) bool { av := reflect.ValueOf(a) @@ -115,6 +130,8 @@ func trimRightSpace(s string) string { return strings.TrimRightFunc(s, unicode.IsSpace) } +// FIXME appendIfNotPresent is unused by cobra and should be removed in a version 2. It exists only for compatibility with users of cobra. + // appendIfNotPresent will append stringToAppend to the end of s, but only if it's not yet present in s. func appendIfNotPresent(s, stringToAppend string) string { if strings.Contains(s, stringToAppend) { @@ -172,3 +189,12 @@ func ld(s, t string, ignoreCase bool) int { } return d[len(s)][len(t)] } + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} diff --git a/vendor/github.com/spf13/cobra/command.go b/vendor/github.com/spf13/cobra/command.go index f4cd95ca..34d1bf36 100644 --- a/vendor/github.com/spf13/cobra/command.go +++ b/vendor/github.com/spf13/cobra/command.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//Package cobra is a commander providing a simple interface to create powerful modern CLI interfaces. -//In addition to providing an interface, Cobra simultaneously provides a controller to organize your application code. +// Package cobra is a commander providing a simple interface to create powerful modern CLI interfaces. +// In addition to providing an interface, Cobra simultaneously provides a controller to organize your application code. package cobra import ( @@ -27,109 +27,174 @@ import ( flag "github.com/spf13/pflag" ) +// FParseErrWhitelist configures Flag parse errors to be ignored +type FParseErrWhitelist flag.ParseErrorsWhitelist + // Command is just that, a command for your application. -// eg. 'go run' ... 'run' is the command. Cobra requires +// E.g. 'go run ...' - 'run' is the command. Cobra requires // you to define the usage and description as part of your command // definition to ensure usability. type Command struct { - // Name is the command name, usually the executable's name. - name string - // The one-line usage message. + // Use is the one-line usage message. Use string - // An array of aliases that can be used instead of the first word in Use. + + // Aliases is an array of aliases that can be used instead of the first word in Use. Aliases []string - // An array of command names for which this command will be suggested - similar to aliases but only suggests. + + // SuggestFor is an array of command names for which this command will be suggested - + // similar to aliases but only suggests. SuggestFor []string - // The short description shown in the 'help' output. + + // Short is the short description shown in the 'help' output. Short string - // The long message shown in the 'help ' output. + + // Long is the long message shown in the 'help ' output. Long string - // Examples of how to use the command + + // Example is examples of how to use the command. Example string - // List of all valid non-flag arguments that are accepted in bash completions + + // ValidArgs is list of all valid non-flag arguments that are accepted in bash completions ValidArgs []string - // List of aliases for ValidArgs. These are not suggested to the user in the bash - // completion, but accepted if entered manually. + + // Expected arguments + Args PositionalArgs + + // ArgAliases is List of aliases for ValidArgs. + // These are not suggested to the user in the bash completion, + // but accepted if entered manually. ArgAliases []string - // Custom functions used by the bash autocompletion generator + + // BashCompletionFunction is custom functions used by the bash autocompletion generator. BashCompletionFunction string - // Is this command deprecated and should print this string when used? + + // Deprecated defines, if this command is deprecated and should print this string when used. Deprecated string - // Is this command hidden and should NOT show up in the list of available commands? + + // Hidden defines, if this command is hidden and should NOT show up in the list of available commands. Hidden bool + // Annotations are key/value pairs that can be used by applications to identify or - // group commands + // group commands. Annotations map[string]string - // Full set of flags - flags *flag.FlagSet - // Set of flags childrens of this command will inherit - pflags *flag.FlagSet - // Flags that are declared specifically by this command (not inherited). - lflags *flag.FlagSet - // SilenceErrors is an option to quiet errors down stream - SilenceErrors bool - // Silence Usage is an option to silence usage when an error occurs. - SilenceUsage bool + + // Version defines the version for this command. If this value is non-empty and the command does not + // define a "version" flag, a "version" boolean flag will be added to the command and, if specified, + // will print content of the "Version" variable. + Version string + // The *Run functions are executed in the following order: // * PersistentPreRun() // * PreRun() // * Run() // * PostRun() // * PersistentPostRun() - // All functions get the same args, the arguments after the command name - // PersistentPreRun: children of this command will inherit and execute + // All functions get the same args, the arguments after the command name. + // + // PersistentPreRun: children of this command will inherit and execute. PersistentPreRun func(cmd *Command, args []string) - // PersistentPreRunE: PersistentPreRun but returns an error + // PersistentPreRunE: PersistentPreRun but returns an error. PersistentPreRunE func(cmd *Command, args []string) error // PreRun: children of this command will not inherit. PreRun func(cmd *Command, args []string) - // PreRunE: PreRun but returns an error + // PreRunE: PreRun but returns an error. PreRunE func(cmd *Command, args []string) error - // Run: Typically the actual work function. Most commands will only implement this + // Run: Typically the actual work function. Most commands will only implement this. Run func(cmd *Command, args []string) - // RunE: Run but returns an error + // RunE: Run but returns an error. RunE func(cmd *Command, args []string) error // PostRun: run after the Run command. PostRun func(cmd *Command, args []string) - // PostRunE: PostRun but returns an error + // PostRunE: PostRun but returns an error. PostRunE func(cmd *Command, args []string) error - // PersistentPostRun: children of this command will inherit and execute after PostRun + // PersistentPostRun: children of this command will inherit and execute after PostRun. PersistentPostRun func(cmd *Command, args []string) - // PersistentPostRunE: PersistentPostRun but returns an error + // PersistentPostRunE: PersistentPostRun but returns an error. PersistentPostRunE func(cmd *Command, args []string) error - // DisableAutoGenTag remove + + // SilenceErrors is an option to quiet errors down stream. + SilenceErrors bool + + // SilenceUsage is an option to silence usage when an error occurs. + SilenceUsage bool + + // DisableFlagParsing disables the flag parsing. + // If this is true all flags will be passed to the command as arguments. + DisableFlagParsing bool + + // DisableAutoGenTag defines, if gen tag ("Auto generated by spf13/cobra...") + // will be printed by generating docs for this command. DisableAutoGenTag bool - // Commands is the list of commands supported by this program. + + // DisableFlagsInUseLine will disable the addition of [flags] to the usage + // line of a command when printing help or generating docs + DisableFlagsInUseLine bool + + // DisableSuggestions disables the suggestions based on Levenshtein distance + // that go along with 'unknown command' messages. + DisableSuggestions bool + // SuggestionsMinimumDistance defines minimum levenshtein distance to display suggestions. + // Must be > 0. + SuggestionsMinimumDistance int + + // TraverseChildren parses flags on all parents before executing child command. + TraverseChildren bool + + //FParseErrWhitelist flag parse errors to be ignored + FParseErrWhitelist FParseErrWhitelist + + // commands is the list of commands supported by this program. commands []*Command - // Parent Command for this command + // parent is a parent command for this command. parent *Command - // max lengths of commands' string lengths for use in padding + // Max lengths of commands' string lengths for use in padding. commandsMaxUseLen int commandsMaxCommandPathLen int commandsMaxNameLen int - // is commands slice are sorted or not + // commandsAreSorted defines, if command slice are sorted or not. commandsAreSorted bool + // commandCalledAs is the name or alias value used to call this command. + commandCalledAs struct { + name string + called bool + } + // args is actual args parsed from flags. + args []string + // flagErrorBuf contains all error messages from pflag. flagErrorBuf *bytes.Buffer - - args []string // actual args parsed from flags - output io.Writer // out writer if set in SetOutput(w) - usageFunc func(*Command) error // Usage can be defined by application - usageTemplate string // Can be defined by Application - flagErrorFunc func(*Command, error) error - helpTemplate string // Can be defined by Application - helpFunc func(*Command, []string) // Help can be defined by application - helpCommand *Command // The help command - // The global normalization function that we can use on every pFlag set and children commands + // flags is full set of flags. + flags *flag.FlagSet + // pflags contains persistent flags. + pflags *flag.FlagSet + // lflags contains local flags. + lflags *flag.FlagSet + // iflags contains inherited flags. + iflags *flag.FlagSet + // parentsPflags is all persistent flags of cmd's parents. + parentsPflags *flag.FlagSet + // globNormFunc is the global normalization function + // that we can use on every pflag set and children commands globNormFunc func(f *flag.FlagSet, name string) flag.NormalizedName - // Disable the suggestions based on Levenshtein distance that go along with 'unknown command' messages - DisableSuggestions bool - // If displaying suggestions, allows to set the minimum levenshtein distance to display, must be > 0 - SuggestionsMinimumDistance int - - // Disable the flag parsing. If this is true all flags will be passed to the command as arguments. - DisableFlagParsing bool + // output is an output writer defined by user. + output io.Writer + // usageFunc is usage func defined by user. + usageFunc func(*Command) error + // usageTemplate is usage template defined by user. + usageTemplate string + // flagErrorFunc is func defined by user and it's called when the parsing of + // flags returns an error. + flagErrorFunc func(*Command, error) error + // helpTemplate is help template defined by user. + helpTemplate string + // helpFunc is help func defined by user. + helpFunc func(*Command, []string) + // helpCommand is command with usage 'help'. If it's not defined by user, + // cobra uses default help command. + helpCommand *Command + // versionTemplate is the version template defined by user. + versionTemplate string } // SetArgs sets arguments for the command. It is set to os.Args[1:] by default, if desired, can be overridden @@ -175,6 +240,11 @@ func (c *Command) SetHelpTemplate(s string) { c.helpTemplate = s } +// SetVersionTemplate sets version template to be used. Application can use it to set custom template. +func (c *Command) SetVersionTemplate(s string) { + c.versionTemplate = s +} + // SetGlobalNormalizationFunc sets a normalization function to all flag sets and also to child commands. // The user should not have a cyclic dependency on commands. func (c *Command) SetGlobalNormalizationFunc(n func(f *flag.FlagSet, name string) flag.NormalizedName) { @@ -213,9 +283,8 @@ func (c *Command) UsageFunc() (f func(*Command) error) { if c.usageFunc != nil { return c.usageFunc } - if c.HasParent() { - return c.parent.UsageFunc() + return c.Parent().UsageFunc() } return func(c *Command) error { c.mergePersistentFlags() @@ -237,10 +306,13 @@ func (c *Command) Usage() error { // HelpFunc returns either the function set by SetHelpFunc for this command // or a parent, or it returns a function with default help behavior. func (c *Command) HelpFunc() func(*Command, []string) { - if helpFunc := c.checkHelpFunc(); helpFunc != nil { - return helpFunc + if c.helpFunc != nil { + return c.helpFunc } - return func(*Command, []string) { + if c.HasParent() { + return c.Parent().HelpFunc() + } + return func(c *Command, a []string) { c.mergePersistentFlags() err := tmpl(c.OutOrStdout(), c.HelpTemplate(), c) if err != nil { @@ -249,20 +321,6 @@ func (c *Command) HelpFunc() func(*Command, []string) { } } -// checkHelpFunc checks if there is helpFunc in ancestors of c. -func (c *Command) checkHelpFunc() func(*Command, []string) { - if c == nil { - return nil - } - if c.helpFunc != nil { - return c.helpFunc - } - if c.HasParent() { - return c.parent.checkHelpFunc() - } - return nil -} - // Help puts out the help for the command. // Used when a user calls help [command]. // Can be defined by user by overriding HelpFunc. @@ -337,23 +395,23 @@ func (c *Command) UsageTemplate() string { return c.parent.UsageTemplate() } return `Usage:{{if .Runnable}} - {{if .HasAvailableFlags}}{{appendIfNotPresent .UseLine "[flags]"}}{{else}}{{.UseLine}}{{end}}{{end}}{{if .HasAvailableSubCommands}} - {{ .CommandPath}} [command]{{end}}{{if gt .Aliases 0}} + {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} + {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} Aliases: {{.NameAndAliases}}{{end}}{{if .HasExample}} Examples: -{{ .Example }}{{end}}{{if .HasAvailableSubCommands}} +{{.Example}}{{end}}{{if .HasAvailableSubCommands}} Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} Flags: -{{.LocalFlags.FlagUsages | trimRightSpace}}{{end}}{{if .HasAvailableInheritedFlags}} +{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} Global Flags: -{{.InheritedFlags.FlagUsages | trimRightSpace}}{{end}}{{if .HasHelpSubCommands}} +{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} @@ -371,70 +429,76 @@ func (c *Command) HelpTemplate() string { if c.HasParent() { return c.parent.HelpTemplate() } - return `{{with or .Long .Short }}{{. | trim}} + return `{{with (or .Long .Short)}}{{. | trimTrailingWhitespaces}} {{end}}{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}}` } -// Really only used when casting a command to a commander. -func (c *Command) resetChildrensParents() { - for _, x := range c.commands { - x.parent = c +// VersionTemplate return version template for the command. +func (c *Command) VersionTemplate() string { + if c.versionTemplate != "" { + return c.versionTemplate } + + if c.HasParent() { + return c.parent.VersionTemplate() + } + return `{{with .Name}}{{printf "%s " .}}{{end}}{{printf "version %s" .Version}} +` } -func hasNoOptDefVal(name string, f *flag.FlagSet) bool { - flag := f.Lookup(name) +func hasNoOptDefVal(name string, fs *flag.FlagSet) bool { + flag := fs.Lookup(name) if flag == nil { return false } - return len(flag.NoOptDefVal) > 0 + return flag.NoOptDefVal != "" } func shortHasNoOptDefVal(name string, fs *flag.FlagSet) bool { - result := false - fs.VisitAll(func(flag *flag.Flag) { - if flag.Shorthand == name && len(flag.NoOptDefVal) > 0 { - result = true - } - }) - return result + if len(name) == 0 { + return false + } + + flag := fs.ShorthandLookup(name[:1]) + if flag == nil { + return false + } + return flag.NoOptDefVal != "" } func stripFlags(args []string, c *Command) []string { - if len(args) < 1 { + if len(args) == 0 { return args } c.mergePersistentFlags() commands := []string{} + flags := c.Flags() - inQuote := false - inFlag := false - for _, y := range args { - if !inQuote { - switch { - case strings.HasPrefix(y, "\""): - inQuote = true - case strings.Contains(y, "=\""): - inQuote = true - case strings.HasPrefix(y, "--") && !strings.Contains(y, "="): - // TODO: this isn't quite right, we should really check ahead for 'true' or 'false' - inFlag = !hasNoOptDefVal(y[2:], c.Flags()) - case strings.HasPrefix(y, "-") && !strings.Contains(y, "=") && len(y) == 2 && !shortHasNoOptDefVal(y[1:], c.Flags()): - inFlag = true - case inFlag: - inFlag = false - case y == "": - // strip empty commands, as the go tests expect this to be ok.... - case !strings.HasPrefix(y, "-"): - commands = append(commands, y) - inFlag = false +Loop: + for len(args) > 0 { + s := args[0] + args = args[1:] + switch { + case s == "--": + // "--" terminates the flags + break Loop + case strings.HasPrefix(s, "--") && !strings.Contains(s, "=") && !hasNoOptDefVal(s[2:], flags): + // If '--flag arg' then + // delete arg from args. + fallthrough // (do the same as below) + case strings.HasPrefix(s, "-") && !strings.Contains(s, "=") && len(s) == 2 && !shortHasNoOptDefVal(s[1:], flags): + // If '-f arg' then + // delete 'arg' from args or break the loop if len(args) <= 1. + if len(args) <= 1 { + break Loop + } else { + args = args[1:] + continue } - } - - if strings.HasSuffix(y, "\"") && !strings.HasSuffix(y, "\\\"") { - inQuote = false + case s != "" && !strings.HasPrefix(s, "-"): + commands = append(commands, s) } } @@ -455,13 +519,14 @@ func argsMinusFirstX(args []string, x string) []string { return args } +func isFlagArg(arg string) bool { + return ((len(arg) >= 3 && arg[1] == '-') || + (len(arg) >= 2 && arg[0] == '-' && arg[1] != '-')) +} + // Find the target command given the args and command tree // Meant to be run on the highest node. Only searches down. func (c *Command) Find(args []string) (*Command, []string, error) { - if c == nil { - return nil, nil, fmt.Errorf("Called find() on a nil Command") - } - var innerfind func(*Command, []string) (*Command, []string) innerfind = func(c *Command, innerArgs []string) (*Command, []string) { @@ -470,59 +535,100 @@ func (c *Command) Find(args []string) (*Command, []string, error) { return c, innerArgs } nextSubCmd := argsWOflags[0] - matches := make([]*Command, 0) - for _, cmd := range c.commands { - if cmd.Name() == nextSubCmd || cmd.HasAlias(nextSubCmd) { // exact name or alias match - return innerfind(cmd, argsMinusFirstX(innerArgs, nextSubCmd)) - } - if EnablePrefixMatching { - if strings.HasPrefix(cmd.Name(), nextSubCmd) { // prefix match - matches = append(matches, cmd) - } - for _, x := range cmd.Aliases { - if strings.HasPrefix(x, nextSubCmd) { - matches = append(matches, cmd) - } - } - } - } - // only accept a single prefix match - multiple matches would be ambiguous - if len(matches) == 1 { - return innerfind(matches[0], argsMinusFirstX(innerArgs, argsWOflags[0])) + cmd := c.findNext(nextSubCmd) + if cmd != nil { + return innerfind(cmd, argsMinusFirstX(innerArgs, nextSubCmd)) } - return c, innerArgs } commandFound, a := innerfind(c, args) - argsWOflags := stripFlags(a, commandFound) - - // no subcommand, always take args - if !commandFound.HasSubCommands() { - return commandFound, a, nil + if commandFound.Args == nil { + return commandFound, a, legacyArgs(commandFound, stripFlags(a, commandFound)) } - - // root command with subcommands, do subcommand checking - if commandFound == c && len(argsWOflags) > 0 { - suggestionsString := "" - if !c.DisableSuggestions { - if c.SuggestionsMinimumDistance <= 0 { - c.SuggestionsMinimumDistance = 2 - } - if suggestions := c.SuggestionsFor(argsWOflags[0]); len(suggestions) > 0 { - suggestionsString += "\n\nDid you mean this?\n" - for _, s := range suggestions { - suggestionsString += fmt.Sprintf("\t%v\n", s) - } - } - } - return commandFound, a, fmt.Errorf("unknown command %q for %q%s", argsWOflags[0], commandFound.CommandPath(), suggestionsString) - } - return commandFound, a, nil } +func (c *Command) findSuggestions(arg string) string { + if c.DisableSuggestions { + return "" + } + if c.SuggestionsMinimumDistance <= 0 { + c.SuggestionsMinimumDistance = 2 + } + suggestionsString := "" + if suggestions := c.SuggestionsFor(arg); len(suggestions) > 0 { + suggestionsString += "\n\nDid you mean this?\n" + for _, s := range suggestions { + suggestionsString += fmt.Sprintf("\t%v\n", s) + } + } + return suggestionsString +} + +func (c *Command) findNext(next string) *Command { + matches := make([]*Command, 0) + for _, cmd := range c.commands { + if cmd.Name() == next || cmd.HasAlias(next) { + cmd.commandCalledAs.name = next + return cmd + } + if EnablePrefixMatching && cmd.hasNameOrAliasPrefix(next) { + matches = append(matches, cmd) + } + } + + if len(matches) == 1 { + return matches[0] + } + + return nil +} + +// Traverse the command tree to find the command, and parse args for +// each parent. +func (c *Command) Traverse(args []string) (*Command, []string, error) { + flags := []string{} + inFlag := false + + for i, arg := range args { + switch { + // A long flag with a space separated value + case strings.HasPrefix(arg, "--") && !strings.Contains(arg, "="): + // TODO: this isn't quite right, we should really check ahead for 'true' or 'false' + inFlag = !hasNoOptDefVal(arg[2:], c.Flags()) + flags = append(flags, arg) + continue + // A short flag with a space separated value + case strings.HasPrefix(arg, "-") && !strings.Contains(arg, "=") && len(arg) == 2 && !shortHasNoOptDefVal(arg[1:], c.Flags()): + inFlag = true + flags = append(flags, arg) + continue + // The value for a flag + case inFlag: + inFlag = false + flags = append(flags, arg) + continue + // A flag without a value, or with an `=` separated value + case isFlagArg(arg): + flags = append(flags, arg) + continue + } + + cmd := c.findNext(arg) + if cmd == nil { + return c, args, nil + } + + if err := c.ParseFlags(flags); err != nil { + return nil, args, err + } + return cmd.Traverse(args[i+1:]) + } + return c, args, nil +} + // SuggestionsFor provides suggestions for the typedName. func (c *Command) SuggestionsFor(typedName string) []string { suggestions := []string{} @@ -546,38 +652,22 @@ func (c *Command) SuggestionsFor(typedName string) []string { // VisitParents visits all parents of the command and invokes fn on each parent. func (c *Command) VisitParents(fn func(*Command)) { - var traverse func(*Command) *Command - - traverse = func(x *Command) *Command { - if x != c { - fn(x) - } - if x.HasParent() { - return traverse(x.parent) - } - return x + if c.HasParent() { + fn(c.Parent()) + c.Parent().VisitParents(fn) } - traverse(c) } // Root finds root command. func (c *Command) Root() *Command { - var findRoot func(*Command) *Command - - findRoot = func(x *Command) *Command { - if x.HasParent() { - return findRoot(x.parent) - } - return x + if c.HasParent() { + return c.Parent().Root() } - - return findRoot(c) + return c } -// ArgsLenAtDash will return the length of f.Args at the moment when a -- was -// found during arg parsing. This allows your program to know which args were -// before the -- and which came after. (Description from -// https://godoc.org/github.com/spf13/pflag#FlagSet.ArgsLenAtDash). +// ArgsLenAtDash will return the length of c.Flags().Args at the moment +// when a -- was found during args parsing. func (c *Command) ArgsLenAtDash() int { return c.Flags().ArgsLenAtDash() } @@ -591,25 +681,47 @@ func (c *Command) execute(a []string) (err error) { c.Printf("Command %q is deprecated, %s\n", c.Name(), c.Deprecated) } - // initialize help flag as the last point possible to allow for user + // initialize help and version flag at the last point possible to allow for user // overriding - c.initHelpFlag() + c.InitDefaultHelpFlag() + c.InitDefaultVersionFlag() err = c.ParseFlags(a) if err != nil { return c.FlagErrorFunc()(c, err) } - // If help is called, regardless of other flags, return we want help + + // If help is called, regardless of other flags, return we want help. // Also say we need help if the command isn't runnable. helpVal, err := c.Flags().GetBool("help") if err != nil { // should be impossible to get here as we always declare a help - // flag in initHelpFlag() + // flag in InitDefaultHelpFlag() c.Println("\"help\" flag declared as non-bool. Please correct your code") return err } - if helpVal || !c.Runnable() { + if helpVal { + return flag.ErrHelp + } + + // for back-compat, only add version flag behavior if version is defined + if c.Version != "" { + versionVal, err := c.Flags().GetBool("version") + if err != nil { + c.Println("\"version\" flag declared as non-bool. Please correct your code") + return err + } + if versionVal { + err := tmpl(c.OutOrStdout(), c.VersionTemplate(), c) + if err != nil { + c.Println(err) + } + return err + } + } + + if !c.Runnable() { return flag.ErrHelp } @@ -620,6 +732,10 @@ func (c *Command) execute(a []string) (err error) { argWoFlags = a } + if err := c.ValidateArgs(argWoFlags); err != nil { + return err + } + for p := c; p != nil; p = p.Parent() { if p.PersistentPreRunE != nil { if err := p.PersistentPreRunE(c, argWoFlags); err != nil { @@ -639,6 +755,9 @@ func (c *Command) execute(a []string) (err error) { c.PreRun(c, argWoFlags) } + if err := c.validateRequiredFlags(); err != nil { + return err + } if c.RunE != nil { if err := c.RunE(c, argWoFlags); err != nil { return err @@ -674,18 +793,7 @@ func (c *Command) preRun() { } } -func (c *Command) errorMsgFromParse() string { - s := c.flagErrorBuf.String() - - x := strings.Split(s, "\n") - - if len(x) > 0 { - return x[0] - } - return "" -} - -// Execute Call execute to use the args (os.Args[1:] by default) +// Execute uses the args (os.Args[1:] by default) // and run through the command tree finding appropriate matches // for commands and then corresponding flags. func (c *Command) Execute() error { @@ -707,7 +815,7 @@ func (c *Command) ExecuteC() (cmd *Command, err error) { // initialize help as the last point possible to allow for user // overriding - c.initHelpCmd() + c.InitDefaultHelpCmd() var args []string @@ -718,7 +826,12 @@ func (c *Command) ExecuteC() (cmd *Command, err error) { args = c.args } - cmd, flags, err := c.Find(args) + var flags []string + if c.TraverseChildren { + cmd, flags, err = c.Traverse(args) + } else { + cmd, flags, err = c.Find(args) + } if err != nil { // If found parse to a subcommand and then failed, talk about the subcommand if cmd != nil { @@ -730,6 +843,12 @@ func (c *Command) ExecuteC() (cmd *Command, err error) { } return c, err } + + cmd.commandCalledAs.called = true + if cmd.commandCalledAs.name == "" { + cmd.commandCalledAs.name = cmd.Name() + } + err = cmd.execute(flags) if err != nil { // Always show help if requested, even if SilenceErrors is in @@ -750,31 +869,87 @@ func (c *Command) ExecuteC() (cmd *Command, err error) { if !cmd.SilenceUsage && !c.SilenceUsage { c.Println(cmd.UsageString()) } - return cmd, err } - return cmd, nil + return cmd, err } -func (c *Command) initHelpFlag() { - c.mergePersistentFlags() - if c.Flags().Lookup("help") == nil { - c.Flags().BoolP("help", "h", false, "help for "+c.Name()) +func (c *Command) ValidateArgs(args []string) error { + if c.Args == nil { + return nil } + return c.Args(c, args) } -func (c *Command) initHelpCmd() { - if c.helpCommand == nil { - if !c.HasSubCommands() { +func (c *Command) validateRequiredFlags() error { + flags := c.Flags() + missingFlagNames := []string{} + flags.VisitAll(func(pflag *flag.Flag) { + requiredAnnotation, found := pflag.Annotations[BashCompOneRequiredFlag] + if !found { return } + if (requiredAnnotation[0] == "true") && !pflag.Changed { + missingFlagNames = append(missingFlagNames, pflag.Name) + } + }) + if len(missingFlagNames) > 0 { + return fmt.Errorf(`required flag(s) "%s" not set`, strings.Join(missingFlagNames, `", "`)) + } + return nil +} + +// InitDefaultHelpFlag adds default help flag to c. +// It is called automatically by executing the c or by calling help and usage. +// If c already has help flag, it will do nothing. +func (c *Command) InitDefaultHelpFlag() { + c.mergePersistentFlags() + if c.Flags().Lookup("help") == nil { + usage := "help for " + if c.Name() == "" { + usage += "this command" + } else { + usage += c.Name() + } + c.Flags().BoolP("help", "h", false, usage) + } +} + +// InitDefaultVersionFlag adds default version flag to c. +// It is called automatically by executing the c. +// If c already has a version flag, it will do nothing. +// If c.Version is empty, it will do nothing. +func (c *Command) InitDefaultVersionFlag() { + if c.Version == "" { + return + } + + c.mergePersistentFlags() + if c.Flags().Lookup("version") == nil { + usage := "version for " + if c.Name() == "" { + usage += "this command" + } else { + usage += c.Name() + } + c.Flags().Bool("version", false, usage) + } +} + +// InitDefaultHelpCmd adds default help command to c. +// It is called automatically by executing the c or by calling help and usage. +// If c already has help command or c has no subcommands, it will do nothing. +func (c *Command) InitDefaultHelpCmd() { + if !c.HasSubCommands() { + return + } + + if c.helpCommand == nil { c.helpCommand = &Command{ Use: "help [command]", Short: "Help about any command", Long: `Help provides help for any command in the application. - Simply type ` + c.Name() + ` help [path to command] for full details.`, - PersistentPreRun: func(cmd *Command, args []string) {}, - PersistentPostRun: func(cmd *Command, args []string) {}, +Simply type ` + c.Name() + ` help [path to command] for full details.`, Run: func(c *Command, args []string) { cmd, _, e := c.Root().Find(args) @@ -782,6 +957,7 @@ func (c *Command) initHelpCmd() { c.Printf("Unknown help topic %#q\n", args) c.Root().Usage() } else { + cmd.InitDefaultHelpFlag() // make possible 'help' flag to be shown cmd.Help() } }, @@ -791,10 +967,12 @@ func (c *Command) initHelpCmd() { c.AddCommand(c.helpCommand) } -// ResetCommands used for testing. +// ResetCommands delete parent, subcommand and help command from c. func (c *Command) ResetCommands() { + c.parent = nil c.commands = nil c.helpCommand = nil + c.parentsPflags = nil } // Sorts commands by their names. @@ -884,34 +1062,37 @@ func (c *Command) Print(i ...interface{}) { // Println is a convenience method to Println to the defined output, fallback to Stderr if not set. func (c *Command) Println(i ...interface{}) { - str := fmt.Sprintln(i...) - c.Print(str) + c.Print(fmt.Sprintln(i...)) } // Printf is a convenience method to Printf to the defined output, fallback to Stderr if not set. func (c *Command) Printf(format string, i ...interface{}) { - str := fmt.Sprintf(format, i...) - c.Print(str) + c.Print(fmt.Sprintf(format, i...)) } // CommandPath returns the full path to this command. func (c *Command) CommandPath() string { - str := c.Name() - x := c - for x.HasParent() { - str = x.parent.Name() + " " + str - x = x.parent + if c.HasParent() { + return c.Parent().CommandPath() + " " + c.Name() } - return str + return c.Name() } // UseLine puts out the full usage for a given command (including parents). func (c *Command) UseLine() string { - str := "" + var useline string if c.HasParent() { - str = c.parent.CommandPath() + " " + useline = c.parent.CommandPath() + " " + c.Use + } else { + useline = c.Use } - return str + c.Use + if c.DisableFlagsInUseLine { + return useline + } + if c.HasAvailableFlags() && !strings.Contains(useline, "[flags]") { + useline += " [flags]" + } + return useline } // DebugFlags used to determine which flags have been assigned to which commands @@ -926,12 +1107,8 @@ func (c *Command) DebugFlags() { } if x.HasFlags() { x.flags.VisitAll(func(f *flag.Flag) { - if x.HasPersistentFlags() { - if x.persistentFlag(f.Name) == nil { - c.Println(" -"+f.Shorthand+",", "--"+f.Name, "["+f.DefValue+"]", "", f.Value, " [L]") - } else { - c.Println(" -"+f.Shorthand+",", "--"+f.Name, "["+f.DefValue+"]", "", f.Value, " [LP]") - } + if x.HasPersistentFlags() && x.persistentFlag(f.Name) != nil { + c.Println(" -"+f.Shorthand+",", "--"+f.Name, "["+f.DefValue+"]", "", f.Value, " [LP]") } else { c.Println(" -"+f.Shorthand+",", "--"+f.Name, "["+f.DefValue+"]", "", f.Value, " [L]") } @@ -961,16 +1138,12 @@ func (c *Command) DebugFlags() { // Name returns the command's name: the first word in the use line. func (c *Command) Name() string { - if c.name != "" { - return c.name - } name := c.Use i := strings.Index(name, " ") if i >= 0 { name = name[:i] } - c.name = name - return c.name + return name } // HasAlias determines if a given string is an alias of the command. @@ -983,7 +1156,32 @@ func (c *Command) HasAlias(s string) bool { return false } -// NameAndAliases returns string containing name and all aliases +// CalledAs returns the command name or alias that was used to invoke +// this command or an empty string if the command has not been called. +func (c *Command) CalledAs() string { + if c.commandCalledAs.called { + return c.commandCalledAs.name + } + return "" +} + +// hasNameOrAliasPrefix returns true if the Name or any of aliases start +// with prefix +func (c *Command) hasNameOrAliasPrefix(prefix string) bool { + if strings.HasPrefix(c.Name(), prefix) { + c.commandCalledAs.name = c.Name() + return true + } + for _, alias := range c.Aliases { + if strings.HasPrefix(alias, prefix) { + c.commandCalledAs.name = alias + return true + } + } + return false +} + +// NameAndAliases returns a list of the command name and all aliases func (c *Command) NameAndAliases() string { return strings.Join(append([]string{c.Name()}, c.Aliases...), ", ") } @@ -1069,7 +1267,7 @@ func (c *Command) HasAvailableSubCommands() bool { } } - // the command either has no sub comamnds, or no available (non deprecated/help/hidden) + // the command either has no sub commands, or no available (non deprecated/help/hidden) // sub commands return false } @@ -1079,7 +1277,7 @@ func (c *Command) HasParent() bool { return c.parent != nil } -// GlobalNormalizationFunc returns the global normalization function or nil if doesn't exists. +// GlobalNormalizationFunc returns the global normalization function or nil if it doesn't exist. func (c *Command) GlobalNormalizationFunc() func(f *flag.FlagSet, name string) flag.NormalizedName { return c.globNormFunc } @@ -1094,6 +1292,7 @@ func (c *Command) Flags() *flag.FlagSet { } c.flags.SetOutput(c.flagErrorBuf) } + return c.flags } @@ -1114,47 +1313,51 @@ func (c *Command) LocalNonPersistentFlags() *flag.FlagSet { func (c *Command) LocalFlags() *flag.FlagSet { c.mergePersistentFlags() - local := flag.NewFlagSet(c.Name(), flag.ContinueOnError) - c.lflags.VisitAll(func(f *flag.Flag) { - local.AddFlag(f) - }) - if !c.HasParent() { - flag.CommandLine.VisitAll(func(f *flag.Flag) { - if local.Lookup(f.Name) == nil { - local.AddFlag(f) - } - }) + if c.lflags == nil { + c.lflags = flag.NewFlagSet(c.Name(), flag.ContinueOnError) + if c.flagErrorBuf == nil { + c.flagErrorBuf = new(bytes.Buffer) + } + c.lflags.SetOutput(c.flagErrorBuf) } - return local + c.lflags.SortFlags = c.Flags().SortFlags + if c.globNormFunc != nil { + c.lflags.SetNormalizeFunc(c.globNormFunc) + } + + addToLocal := func(f *flag.Flag) { + if c.lflags.Lookup(f.Name) == nil && c.parentsPflags.Lookup(f.Name) == nil { + c.lflags.AddFlag(f) + } + } + c.Flags().VisitAll(addToLocal) + c.PersistentFlags().VisitAll(addToLocal) + return c.lflags } // InheritedFlags returns all flags which were inherited from parents commands. func (c *Command) InheritedFlags() *flag.FlagSet { c.mergePersistentFlags() - inherited := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + if c.iflags == nil { + c.iflags = flag.NewFlagSet(c.Name(), flag.ContinueOnError) + if c.flagErrorBuf == nil { + c.flagErrorBuf = new(bytes.Buffer) + } + c.iflags.SetOutput(c.flagErrorBuf) + } + local := c.LocalFlags() - - var rmerge func(x *Command) - - rmerge = func(x *Command) { - if x.HasPersistentFlags() { - x.PersistentFlags().VisitAll(func(f *flag.Flag) { - if inherited.Lookup(f.Name) == nil && local.Lookup(f.Name) == nil { - inherited.AddFlag(f) - } - }) - } - if x.HasParent() { - rmerge(x.parent) - } + if c.globNormFunc != nil { + c.iflags.SetNormalizeFunc(c.globNormFunc) } - if c.HasParent() { - rmerge(c.parent) - } - - return inherited + c.parentsPflags.VisitAll(func(f *flag.Flag) { + if c.iflags.Lookup(f.Name) == nil && local.Lookup(f.Name) == nil { + c.iflags.AddFlag(f) + } + }) + return c.iflags } // NonInheritedFlags returns all flags which were not inherited from parent commands. @@ -1174,7 +1377,7 @@ func (c *Command) PersistentFlags() *flag.FlagSet { return c.pflags } -// ResetFlags is used in testing. +// ResetFlags deletes all flags from command. func (c *Command) ResetFlags() { c.flagErrorBuf = new(bytes.Buffer) c.flagErrorBuf.Reset() @@ -1182,6 +1385,10 @@ func (c *Command) ResetFlags() { c.flags.SetOutput(c.flagErrorBuf) c.pflags = flag.NewFlagSet(c.Name(), flag.ContinueOnError) c.pflags.SetOutput(c.flagErrorBuf) + + c.lflags = nil + c.iflags = nil + c.parentsPflags = nil } // HasFlags checks if the command contains any flags (local plus persistent from the entire structure). @@ -1244,20 +1451,35 @@ func (c *Command) persistentFlag(name string) (flag *flag.Flag) { flag = c.PersistentFlags().Lookup(name) } - if flag == nil && c.HasParent() { - flag = c.parent.persistentFlag(name) + if flag == nil { + c.updateParentsPflags() + flag = c.parentsPflags.Lookup(name) } return } // ParseFlags parses persistent flag tree and local flags. -func (c *Command) ParseFlags(args []string) (err error) { +func (c *Command) ParseFlags(args []string) error { if c.DisableFlagParsing { return nil } + + if c.flagErrorBuf == nil { + c.flagErrorBuf = new(bytes.Buffer) + } + beforeErrorBufLen := c.flagErrorBuf.Len() c.mergePersistentFlags() - err = c.Flags().Parse(args) - return + + //do it here after merging all flags and just before parse + c.Flags().ParseErrorsWhitelist = flag.ParseErrorsWhitelist(c.FParseErrWhitelist) + + err := c.Flags().Parse(args) + // Print warnings if they occurred (e.g. deprecated flag messages). + if c.flagErrorBuf.Len()-beforeErrorBufLen > 0 && err == nil { + c.Print(c.flagErrorBuf.String()) + } + + return err } // Parent returns a commands parent command. @@ -1265,41 +1487,31 @@ func (c *Command) Parent() *Command { return c.parent } +// mergePersistentFlags merges c.PersistentFlags() to c.Flags() +// and adds missing persistent flags of all parents. func (c *Command) mergePersistentFlags() { - var rmerge func(x *Command) - - // Save the set of local flags - if c.lflags == nil { - c.lflags = flag.NewFlagSet(c.Name(), flag.ContinueOnError) - if c.flagErrorBuf == nil { - c.flagErrorBuf = new(bytes.Buffer) - } - c.lflags.SetOutput(c.flagErrorBuf) - addtolocal := func(f *flag.Flag) { - c.lflags.AddFlag(f) - } - c.Flags().VisitAll(addtolocal) - c.PersistentFlags().VisitAll(addtolocal) - } - rmerge = func(x *Command) { - if !x.HasParent() { - flag.CommandLine.VisitAll(func(f *flag.Flag) { - if x.PersistentFlags().Lookup(f.Name) == nil { - x.PersistentFlags().AddFlag(f) - } - }) - } - if x.HasPersistentFlags() { - x.PersistentFlags().VisitAll(func(f *flag.Flag) { - if c.Flags().Lookup(f.Name) == nil { - c.Flags().AddFlag(f) - } - }) - } - if x.HasParent() { - rmerge(x.parent) - } - } - - rmerge(c) + c.updateParentsPflags() + c.Flags().AddFlagSet(c.PersistentFlags()) + c.Flags().AddFlagSet(c.parentsPflags) +} + +// updateParentsPflags updates c.parentsPflags by adding +// new persistent flags of all parents. +// If c.parentsPflags == nil, it makes new. +func (c *Command) updateParentsPflags() { + if c.parentsPflags == nil { + c.parentsPflags = flag.NewFlagSet(c.Name(), flag.ContinueOnError) + c.parentsPflags.SetOutput(c.flagErrorBuf) + c.parentsPflags.SortFlags = false + } + + if c.globNormFunc != nil { + c.parentsPflags.SetNormalizeFunc(c.globNormFunc) + } + + c.Root().PersistentFlags().AddFlagSet(flag.CommandLine) + + c.VisitParents(func(parent *Command) { + c.parentsPflags.AddFlagSet(parent.PersistentFlags()) + }) } diff --git a/vendor/github.com/spf13/cobra/command_win.go b/vendor/github.com/spf13/cobra/command_win.go index 4b0eaa1b..edec728e 100644 --- a/vendor/github.com/spf13/cobra/command_win.go +++ b/vendor/github.com/spf13/cobra/command_win.go @@ -11,14 +11,8 @@ import ( var preExecHookFn = preExecHook -// enables an information splash screen on Windows if the CLI is started from explorer.exe. -var MousetrapHelpText string = `This is a command line tool - -You need to open cmd.exe and run it from there. -` - func preExecHook(c *Command) { - if mousetrap.StartedByExplorer() { + if MousetrapHelpText != "" && mousetrap.StartedByExplorer() { c.Print(MousetrapHelpText) time.Sleep(5 * time.Second) os.Exit(1) diff --git a/vendor/github.com/spf13/cobra/zsh_completions.go b/vendor/github.com/spf13/cobra/zsh_completions.go new file mode 100644 index 00000000..889c22e2 --- /dev/null +++ b/vendor/github.com/spf13/cobra/zsh_completions.go @@ -0,0 +1,126 @@ +package cobra + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" +) + +// GenZshCompletionFile generates zsh completion file. +func (c *Command) GenZshCompletionFile(filename string) error { + outFile, err := os.Create(filename) + if err != nil { + return err + } + defer outFile.Close() + + return c.GenZshCompletion(outFile) +} + +// GenZshCompletion generates a zsh completion file and writes to the passed writer. +func (c *Command) GenZshCompletion(w io.Writer) error { + buf := new(bytes.Buffer) + + writeHeader(buf, c) + maxDepth := maxDepth(c) + writeLevelMapping(buf, maxDepth) + writeLevelCases(buf, maxDepth, c) + + _, err := buf.WriteTo(w) + return err +} + +func writeHeader(w io.Writer, cmd *Command) { + fmt.Fprintf(w, "#compdef %s\n\n", cmd.Name()) +} + +func maxDepth(c *Command) int { + if len(c.Commands()) == 0 { + return 0 + } + maxDepthSub := 0 + for _, s := range c.Commands() { + subDepth := maxDepth(s) + if subDepth > maxDepthSub { + maxDepthSub = subDepth + } + } + return 1 + maxDepthSub +} + +func writeLevelMapping(w io.Writer, numLevels int) { + fmt.Fprintln(w, `_arguments \`) + for i := 1; i <= numLevels; i++ { + fmt.Fprintf(w, ` '%d: :->level%d' \`, i, i) + fmt.Fprintln(w) + } + fmt.Fprintf(w, ` '%d: :%s'`, numLevels+1, "_files") + fmt.Fprintln(w) +} + +func writeLevelCases(w io.Writer, maxDepth int, root *Command) { + fmt.Fprintln(w, "case $state in") + defer fmt.Fprintln(w, "esac") + + for i := 1; i <= maxDepth; i++ { + fmt.Fprintf(w, " level%d)\n", i) + writeLevel(w, root, i) + fmt.Fprintln(w, " ;;") + } + fmt.Fprintln(w, " *)") + fmt.Fprintln(w, " _arguments '*: :_files'") + fmt.Fprintln(w, " ;;") +} + +func writeLevel(w io.Writer, root *Command, i int) { + fmt.Fprintf(w, " case $words[%d] in\n", i) + defer fmt.Fprintln(w, " esac") + + commands := filterByLevel(root, i) + byParent := groupByParent(commands) + + for p, c := range byParent { + names := names(c) + fmt.Fprintf(w, " %s)\n", p) + fmt.Fprintf(w, " _arguments '%d: :(%s)'\n", i, strings.Join(names, " ")) + fmt.Fprintln(w, " ;;") + } + fmt.Fprintln(w, " *)") + fmt.Fprintln(w, " _arguments '*: :_files'") + fmt.Fprintln(w, " ;;") + +} + +func filterByLevel(c *Command, l int) []*Command { + cs := make([]*Command, 0) + if l == 0 { + cs = append(cs, c) + return cs + } + for _, s := range c.Commands() { + cs = append(cs, filterByLevel(s, l-1)...) + } + return cs +} + +func groupByParent(commands []*Command) map[string][]*Command { + m := make(map[string][]*Command) + for _, c := range commands { + parent := c.Parent() + if parent == nil { + continue + } + m[parent.Name()] = append(m[parent.Name()], c) + } + return m +} + +func names(commands []*Command) []string { + ns := make([]string, len(commands)) + for i, c := range commands { + ns[i] = c.Name() + } + return ns +} diff --git a/vendor/github.com/spf13/jwalterweatherman/README.md b/vendor/github.com/spf13/jwalterweatherman/README.md index 350a9683..d8cfd27a 100644 --- a/vendor/github.com/spf13/jwalterweatherman/README.md +++ b/vendor/github.com/spf13/jwalterweatherman/README.md @@ -144,5 +144,5 @@ This is an early release. I’ve been using it for a while and this is the third interface I’ve tried. I like this one pretty well, but no guarantees that it won’t change a bit. -I wrote this for use in [hugo](http://hugo.spf13.com). If you are looking +I wrote this for use in [hugo](https://gohugo.io). If you are looking for a static website engine that’s super fast please checkout Hugo. diff --git a/vendor/github.com/spf13/jwalterweatherman/log_counter.go b/vendor/github.com/spf13/jwalterweatherman/log_counter.go index 570db1d4..11423ac4 100644 --- a/vendor/github.com/spf13/jwalterweatherman/log_counter.go +++ b/vendor/github.com/spf13/jwalterweatherman/log_counter.go @@ -27,7 +27,6 @@ func (c *logCounter) getCount() uint64 { func (c *logCounter) Write(p []byte) (n int, err error) { c.incr() - return len(p), nil } diff --git a/vendor/github.com/spf13/jwalterweatherman/notepad.go b/vendor/github.com/spf13/jwalterweatherman/notepad.go index 5a623f48..ae5aaf71 100644 --- a/vendor/github.com/spf13/jwalterweatherman/notepad.go +++ b/vendor/github.com/spf13/jwalterweatherman/notepad.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "log" - "os" ) type Threshold int @@ -38,11 +37,7 @@ var prefixes map[Threshold]string = map[Threshold]string{ LevelFatal: "FATAL", } -func prefix(t Threshold) string { - return t.String() + " " -} - -// Notepad is where you leave a note ! +// Notepad is where you leave a note! type Notepad struct { TRACE *log.Logger DEBUG *log.Logger @@ -55,7 +50,7 @@ type Notepad struct { LOG *log.Logger FEEDBACK *Feedback - loggers []**log.Logger + loggers [7]**log.Logger logHandle io.Writer outHandle io.Writer logThreshold Threshold @@ -71,11 +66,11 @@ type Notepad struct { func NewNotepad(outThreshold Threshold, logThreshold Threshold, outHandle, logHandle io.Writer, prefix string, flags int) *Notepad { n := &Notepad{} - n.loggers = append(n.loggers, &n.TRACE, &n.DEBUG, &n.INFO, &n.WARN, &n.ERROR, &n.CRITICAL, &n.FATAL) - n.logHandle = logHandle + n.loggers = [7]**log.Logger{&n.TRACE, &n.DEBUG, &n.INFO, &n.WARN, &n.ERROR, &n.CRITICAL, &n.FATAL} n.outHandle = outHandle - n.logThreshold = logThreshold + n.logHandle = logHandle n.stdoutThreshold = outThreshold + n.logThreshold = logThreshold if len(prefix) != 0 { n.prefix = "[" + prefix + "] " @@ -88,54 +83,48 @@ func NewNotepad(outThreshold Threshold, logThreshold Threshold, outHandle, logHa n.LOG = log.New(n.logHandle, "LOG: ", n.flags) - - n.FEEDBACK = &Feedback{n} + n.FEEDBACK = &Feedback{out: log.New(outHandle, "", 0), log: n.LOG} n.init() - return n } -// Feedback is special. It writes plainly to the output while -// logging with the standard extra information (date, file, etc) -// Only Println and Printf are currently provided for this -type Feedback struct { - *Notepad -} - -// init create the loggers for each level depending on the notepad thresholds +// init creates the loggers for each level depending on the notepad thresholds. func (n *Notepad) init() { - bothHandle := io.MultiWriter(n.outHandle, n.logHandle) + logAndOut := io.MultiWriter(n.outHandle, n.logHandle) for t, logger := range n.loggers { threshold := Threshold(t) counter := &logCounter{} n.logCounters[t] = counter + prefix := n.prefix + threshold.String() + " " switch { case threshold >= n.logThreshold && threshold >= n.stdoutThreshold: - *logger = log.New(io.MultiWriter(counter, bothHandle), n.prefix+prefix(threshold), n.flags) + *logger = log.New(io.MultiWriter(counter, logAndOut), prefix, n.flags) case threshold >= n.logThreshold: - *logger = log.New(io.MultiWriter(counter, n.logHandle), n.prefix+prefix(threshold), n.flags) + *logger = log.New(io.MultiWriter(counter, n.logHandle), prefix, n.flags) case threshold >= n.stdoutThreshold: - *logger = log.New(io.MultiWriter(counter, os.Stdout), n.prefix+prefix(threshold), n.flags) + *logger = log.New(io.MultiWriter(counter, n.outHandle), prefix, n.flags) default: - *logger = log.New(counter, n.prefix+prefix(threshold), n.flags) + // counter doesn't care about prefix and flags, so don't use them + // for performance. + *logger = log.New(counter, "", 0) } } } -// SetLogThreshold change the threshold above which messages are written to the -// log file +// SetLogThreshold changes the threshold above which messages are written to the +// log file. func (n *Notepad) SetLogThreshold(threshold Threshold) { n.logThreshold = threshold n.init() } -// SetLogOutput change the file where log messages are written +// SetLogOutput changes the file where log messages are written. func (n *Notepad) SetLogOutput(handle io.Writer) { n.logHandle = handle n.init() @@ -146,8 +135,8 @@ func (n *Notepad) GetLogThreshold() Threshold { return n.logThreshold } -// SetStdoutThreshold change the threshold above which messages are written to the -// standard output +// SetStdoutThreshold changes the threshold above which messages are written to the +// standard output. func (n *Notepad) SetStdoutThreshold(threshold Threshold) { n.stdoutThreshold = threshold n.init() @@ -158,8 +147,8 @@ func (n *Notepad) GetStdoutThreshold() Threshold { return n.stdoutThreshold } -// SetPrefix change the prefix used by the notepad. Prefixes are displayed between -// brackets at the begining of the line. An empty prefix won't be displayed at all. +// SetPrefix changes the prefix used by the notepad. Prefixes are displayed between +// brackets at the beginning of the line. An empty prefix won't be displayed at all. func (n *Notepad) SetPrefix(prefix string) { if len(prefix) != 0 { n.prefix = "[" + prefix + "] " @@ -176,20 +165,30 @@ func (n *Notepad) SetFlags(flags int) { n.init() } -// Feedback is special. It writes plainly to the output while -// logging with the standard extra information (date, file, etc) -// Only Println and Printf are currently provided for this -func (fb *Feedback) Println(v ...interface{}) { - s := fmt.Sprintln(v...) - fmt.Print(s) - fb.LOG.Output(2, s) +// Feedback writes plainly to the outHandle while +// logging with the standard extra information (date, file, etc). +type Feedback struct { + out *log.Logger + log *log.Logger } -// Feedback is special. It writes plainly to the output while -// logging with the standard extra information (date, file, etc) -// Only Println and Printf are currently provided for this -func (fb *Feedback) Printf(format string, v ...interface{}) { - s := fmt.Sprintf(format, v...) - fmt.Print(s) - fb.LOG.Output(2, s) +func (fb *Feedback) Println(v ...interface{}) { + fb.output(fmt.Sprintln(v...)) +} + +func (fb *Feedback) Printf(format string, v ...interface{}) { + fb.output(fmt.Sprintf(format, v...)) +} + +func (fb *Feedback) Print(v ...interface{}) { + fb.output(fmt.Sprint(v...)) +} + +func (fb *Feedback) output(s string) { + if fb.out != nil { + fb.out.Output(2, s) + } + if fb.log != nil { + fb.log.Output(2, s) + } } diff --git a/vendor/github.com/spf13/pflag/README.md b/vendor/github.com/spf13/pflag/README.md index 3001fcd2..b052414d 100644 --- a/vendor/github.com/spf13/pflag/README.md +++ b/vendor/github.com/spf13/pflag/README.md @@ -246,6 +246,25 @@ It is possible to mark a flag as hidden, meaning it will still function as norma flags.MarkHidden("secretFlag") ``` +## Disable sorting of flags +`pflag` allows you to disable sorting of flags for help and usage message. + +**Example**: +```go +flags.BoolP("verbose", "v", false, "verbose output") +flags.String("coolflag", "yeaah", "it's really cool flag") +flags.Int("usefulflag", 777, "sometimes it's very useful") +flags.SortFlags = false +flags.PrintDefaults() +``` +**Output**: +``` + -v, --verbose verbose output + --coolflag string it's really cool flag (default "yeaah") + --usefulflag int sometimes it's very useful (default 777) +``` + + ## Supporting Go flags when using pflag In order to support flags defined using Go's `flag` package, they must be added to the `pflag` flagset. This is usually necessary to support flags defined by third-party dependencies (e.g. `golang/glog`). diff --git a/vendor/github.com/spf13/pflag/bytes.go b/vendor/github.com/spf13/pflag/bytes.go new file mode 100644 index 00000000..12c58db9 --- /dev/null +++ b/vendor/github.com/spf13/pflag/bytes.go @@ -0,0 +1,105 @@ +package pflag + +import ( + "encoding/hex" + "fmt" + "strings" +) + +// BytesHex adapts []byte for use as a flag. Value of flag is HEX encoded +type bytesHexValue []byte + +func (bytesHex bytesHexValue) String() string { + return fmt.Sprintf("%X", []byte(bytesHex)) +} + +func (bytesHex *bytesHexValue) Set(value string) error { + bin, err := hex.DecodeString(strings.TrimSpace(value)) + + if err != nil { + return err + } + + *bytesHex = bin + + return nil +} + +func (*bytesHexValue) Type() string { + return "bytesHex" +} + +func newBytesHexValue(val []byte, p *[]byte) *bytesHexValue { + *p = val + return (*bytesHexValue)(p) +} + +func bytesHexConv(sval string) (interface{}, error) { + + bin, err := hex.DecodeString(sval) + + if err == nil { + return bin, nil + } + + return nil, fmt.Errorf("invalid string being converted to Bytes: %s %s", sval, err) +} + +// GetBytesHex return the []byte value of a flag with the given name +func (f *FlagSet) GetBytesHex(name string) ([]byte, error) { + val, err := f.getFlagType(name, "bytesHex", bytesHexConv) + + if err != nil { + return []byte{}, err + } + + return val.([]byte), nil +} + +// BytesHexVar defines an []byte flag with specified name, default value, and usage string. +// The argument p points to an []byte variable in which to store the value of the flag. +func (f *FlagSet) BytesHexVar(p *[]byte, name string, value []byte, usage string) { + f.VarP(newBytesHexValue(value, p), name, "", usage) +} + +// BytesHexVarP is like BytesHexVar, but accepts a shorthand letter that can be used after a single dash. +func (f *FlagSet) BytesHexVarP(p *[]byte, name, shorthand string, value []byte, usage string) { + f.VarP(newBytesHexValue(value, p), name, shorthand, usage) +} + +// BytesHexVar defines an []byte flag with specified name, default value, and usage string. +// The argument p points to an []byte variable in which to store the value of the flag. +func BytesHexVar(p *[]byte, name string, value []byte, usage string) { + CommandLine.VarP(newBytesHexValue(value, p), name, "", usage) +} + +// BytesHexVarP is like BytesHexVar, but accepts a shorthand letter that can be used after a single dash. +func BytesHexVarP(p *[]byte, name, shorthand string, value []byte, usage string) { + CommandLine.VarP(newBytesHexValue(value, p), name, shorthand, usage) +} + +// BytesHex defines an []byte flag with specified name, default value, and usage string. +// The return value is the address of an []byte variable that stores the value of the flag. +func (f *FlagSet) BytesHex(name string, value []byte, usage string) *[]byte { + p := new([]byte) + f.BytesHexVarP(p, name, "", value, usage) + return p +} + +// BytesHexP is like BytesHex, but accepts a shorthand letter that can be used after a single dash. +func (f *FlagSet) BytesHexP(name, shorthand string, value []byte, usage string) *[]byte { + p := new([]byte) + f.BytesHexVarP(p, name, shorthand, value, usage) + return p +} + +// BytesHex defines an []byte flag with specified name, default value, and usage string. +// The return value is the address of an []byte variable that stores the value of the flag. +func BytesHex(name string, value []byte, usage string) *[]byte { + return CommandLine.BytesHexP(name, "", value, usage) +} + +// BytesHexP is like BytesHex, but accepts a shorthand letter that can be used after a single dash. +func BytesHexP(name, shorthand string, value []byte, usage string) *[]byte { + return CommandLine.BytesHexP(name, shorthand, value, usage) +} diff --git a/vendor/github.com/spf13/pflag/count.go b/vendor/github.com/spf13/pflag/count.go index d22be41f..aa126e44 100644 --- a/vendor/github.com/spf13/pflag/count.go +++ b/vendor/github.com/spf13/pflag/count.go @@ -11,13 +11,13 @@ func newCountValue(val int, p *int) *countValue { } func (i *countValue) Set(s string) error { - v, err := strconv.ParseInt(s, 0, 64) - // -1 means that no specific value was passed, so increment - if v == -1 { + // "+1" means that no specific value was passed, so increment + if s == "+1" { *i = countValue(*i + 1) - } else { - *i = countValue(v) + return nil } + v, err := strconv.ParseInt(s, 0, 0) + *i = countValue(v) return err } @@ -54,7 +54,7 @@ func (f *FlagSet) CountVar(p *int, name string, usage string) { // CountVarP is like CountVar only take a shorthand for the flag name. func (f *FlagSet) CountVarP(p *int, name, shorthand string, usage string) { flag := f.VarPF(newCountValue(0, p), name, shorthand, usage) - flag.NoOptDefVal = "-1" + flag.NoOptDefVal = "+1" } // CountVar like CountVar only the flag is placed on the CommandLine instead of a given flag set @@ -83,7 +83,9 @@ func (f *FlagSet) CountP(name, shorthand string, usage string) *int { return p } -// Count like Count only the flag is placed on the CommandLine isntead of a given flag set +// Count defines a count flag with specified name, default value, and usage string. +// The return value is the address of an int variable that stores the value of the flag. +// A count flag will add 1 to its value evey time it is found on the command line func Count(name string, usage string) *int { return CommandLine.CountP(name, "", usage) } diff --git a/vendor/github.com/spf13/pflag/duration_slice.go b/vendor/github.com/spf13/pflag/duration_slice.go new file mode 100644 index 00000000..52c6b6dc --- /dev/null +++ b/vendor/github.com/spf13/pflag/duration_slice.go @@ -0,0 +1,128 @@ +package pflag + +import ( + "fmt" + "strings" + "time" +) + +// -- durationSlice Value +type durationSliceValue struct { + value *[]time.Duration + changed bool +} + +func newDurationSliceValue(val []time.Duration, p *[]time.Duration) *durationSliceValue { + dsv := new(durationSliceValue) + dsv.value = p + *dsv.value = val + return dsv +} + +func (s *durationSliceValue) Set(val string) error { + ss := strings.Split(val, ",") + out := make([]time.Duration, len(ss)) + for i, d := range ss { + var err error + out[i], err = time.ParseDuration(d) + if err != nil { + return err + } + + } + if !s.changed { + *s.value = out + } else { + *s.value = append(*s.value, out...) + } + s.changed = true + return nil +} + +func (s *durationSliceValue) Type() string { + return "durationSlice" +} + +func (s *durationSliceValue) String() string { + out := make([]string, len(*s.value)) + for i, d := range *s.value { + out[i] = fmt.Sprintf("%s", d) + } + return "[" + strings.Join(out, ",") + "]" +} + +func durationSliceConv(val string) (interface{}, error) { + val = strings.Trim(val, "[]") + // Empty string would cause a slice with one (empty) entry + if len(val) == 0 { + return []time.Duration{}, nil + } + ss := strings.Split(val, ",") + out := make([]time.Duration, len(ss)) + for i, d := range ss { + var err error + out[i], err = time.ParseDuration(d) + if err != nil { + return nil, err + } + + } + return out, nil +} + +// GetDurationSlice returns the []time.Duration value of a flag with the given name +func (f *FlagSet) GetDurationSlice(name string) ([]time.Duration, error) { + val, err := f.getFlagType(name, "durationSlice", durationSliceConv) + if err != nil { + return []time.Duration{}, err + } + return val.([]time.Duration), nil +} + +// DurationSliceVar defines a durationSlice flag with specified name, default value, and usage string. +// The argument p points to a []time.Duration variable in which to store the value of the flag. +func (f *FlagSet) DurationSliceVar(p *[]time.Duration, name string, value []time.Duration, usage string) { + f.VarP(newDurationSliceValue(value, p), name, "", usage) +} + +// DurationSliceVarP is like DurationSliceVar, but accepts a shorthand letter that can be used after a single dash. +func (f *FlagSet) DurationSliceVarP(p *[]time.Duration, name, shorthand string, value []time.Duration, usage string) { + f.VarP(newDurationSliceValue(value, p), name, shorthand, usage) +} + +// DurationSliceVar defines a duration[] flag with specified name, default value, and usage string. +// The argument p points to a duration[] variable in which to store the value of the flag. +func DurationSliceVar(p *[]time.Duration, name string, value []time.Duration, usage string) { + CommandLine.VarP(newDurationSliceValue(value, p), name, "", usage) +} + +// DurationSliceVarP is like DurationSliceVar, but accepts a shorthand letter that can be used after a single dash. +func DurationSliceVarP(p *[]time.Duration, name, shorthand string, value []time.Duration, usage string) { + CommandLine.VarP(newDurationSliceValue(value, p), name, shorthand, usage) +} + +// DurationSlice defines a []time.Duration flag with specified name, default value, and usage string. +// The return value is the address of a []time.Duration variable that stores the value of the flag. +func (f *FlagSet) DurationSlice(name string, value []time.Duration, usage string) *[]time.Duration { + p := []time.Duration{} + f.DurationSliceVarP(&p, name, "", value, usage) + return &p +} + +// DurationSliceP is like DurationSlice, but accepts a shorthand letter that can be used after a single dash. +func (f *FlagSet) DurationSliceP(name, shorthand string, value []time.Duration, usage string) *[]time.Duration { + p := []time.Duration{} + f.DurationSliceVarP(&p, name, shorthand, value, usage) + return &p +} + +// DurationSlice defines a []time.Duration flag with specified name, default value, and usage string. +// The return value is the address of a []time.Duration variable that stores the value of the flag. +func DurationSlice(name string, value []time.Duration, usage string) *[]time.Duration { + return CommandLine.DurationSliceP(name, "", value, usage) +} + +// DurationSliceP is like DurationSlice, but accepts a shorthand letter that can be used after a single dash. +func DurationSliceP(name, shorthand string, value []time.Duration, usage string) *[]time.Duration { + return CommandLine.DurationSliceP(name, shorthand, value, usage) +} diff --git a/vendor/github.com/spf13/pflag/flag.go b/vendor/github.com/spf13/pflag/flag.go index b9e10c57..5eadc84e 100644 --- a/vendor/github.com/spf13/pflag/flag.go +++ b/vendor/github.com/spf13/pflag/flag.go @@ -101,6 +101,7 @@ package pflag import ( "bytes" "errors" + goflag "flag" "fmt" "io" "os" @@ -123,6 +124,12 @@ const ( PanicOnError ) +// ParseErrorsWhitelist defines the parsing errors that can be ignored +type ParseErrorsWhitelist struct { + // UnknownFlags will ignore unknown flags errors and continue parsing rest of the flags + UnknownFlags bool +} + // NormalizedName is a flag name that has been normalized according to rules // for the FlagSet (e.g. making '-' and '_' equivalent). type NormalizedName string @@ -138,12 +145,17 @@ type FlagSet struct { // help/usage messages. SortFlags bool + // ParseErrorsWhitelist is used to configure a whitelist of errors + ParseErrorsWhitelist ParseErrorsWhitelist + name string parsed bool actual map[NormalizedName]*Flag orderedActual []*Flag + sortedActual []*Flag formal map[NormalizedName]*Flag orderedFormal []*Flag + sortedFormal []*Flag shorthands map[byte]*Flag args []string // arguments after flags argsLenAtDash int // len(args) when a '--' was located when parsing, or -1 if no -- @@ -151,6 +163,8 @@ type FlagSet struct { output io.Writer // nil means stderr; use out() accessor interspersed bool // allow interspersed option/non-option args normalizeNameFunc func(f *FlagSet, name string) NormalizedName + + addedGoFlagSets []*goflag.FlagSet } // A Flag represents the state of a flag. @@ -199,12 +213,19 @@ func sortFlags(flags map[NormalizedName]*Flag) []*Flag { // "--getUrl" which may also be translated to "geturl" and everything will work. func (f *FlagSet) SetNormalizeFunc(n func(f *FlagSet, name string) NormalizedName) { f.normalizeNameFunc = n - for k, v := range f.orderedFormal { - delete(f.formal, NormalizedName(v.Name)) - nname := f.normalizeFlagName(v.Name) - v.Name = string(nname) - f.formal[nname] = v - f.orderedFormal[k] = v + f.sortedFormal = f.sortedFormal[:0] + for fname, flag := range f.formal { + nname := f.normalizeFlagName(flag.Name) + if fname == nname { + continue + } + flag.Name = string(nname) + delete(f.formal, fname) + f.formal[nname] = flag + if _, set := f.actual[fname]; set { + delete(f.actual, fname) + f.actual[nname] = flag + } } } @@ -239,9 +260,16 @@ func (f *FlagSet) SetOutput(output io.Writer) { // in primordial order if f.SortFlags is false, calling fn for each. // It visits all flags, even those not set. func (f *FlagSet) VisitAll(fn func(*Flag)) { + if len(f.formal) == 0 { + return + } + var flags []*Flag if f.SortFlags { - flags = sortFlags(f.formal) + if len(f.formal) != len(f.sortedFormal) { + f.sortedFormal = sortFlags(f.formal) + } + flags = f.sortedFormal } else { flags = f.orderedFormal } @@ -251,16 +279,16 @@ func (f *FlagSet) VisitAll(fn func(*Flag)) { } } -// HasFlags returns a bool to indicate if the FlagSet has any flags definied. +// HasFlags returns a bool to indicate if the FlagSet has any flags defined. func (f *FlagSet) HasFlags() bool { return len(f.formal) > 0 } // HasAvailableFlags returns a bool to indicate if the FlagSet has any flags -// definied that are not hidden or deprecated. +// that are not hidden. func (f *FlagSet) HasAvailableFlags() bool { for _, flag := range f.formal { - if !flag.Hidden && len(flag.Deprecated) == 0 { + if !flag.Hidden { return true } } @@ -278,9 +306,16 @@ func VisitAll(fn func(*Flag)) { // in primordial order if f.SortFlags is false, calling fn for each. // It visits only those flags that have been set. func (f *FlagSet) Visit(fn func(*Flag)) { + if len(f.actual) == 0 { + return + } + var flags []*Flag if f.SortFlags { - flags = sortFlags(f.actual) + if len(f.actual) != len(f.sortedActual) { + f.sortedActual = sortFlags(f.actual) + } + flags = f.sortedActual } else { flags = f.orderedActual } @@ -302,6 +337,22 @@ func (f *FlagSet) Lookup(name string) *Flag { return f.lookup(f.normalizeFlagName(name)) } +// ShorthandLookup returns the Flag structure of the short handed flag, +// returning nil if none exists. +// It panics, if len(name) > 1. +func (f *FlagSet) ShorthandLookup(name string) *Flag { + if name == "" { + return nil + } + if len(name) > 1 { + msg := fmt.Sprintf("can not look up shorthand which is more than one ASCII character: %q", name) + fmt.Fprintf(f.out(), msg) + panic(msg) + } + c := name[0] + return f.shorthands[c] +} + // lookup returns the Flag structure of the named flag, returning nil if none exists. func (f *FlagSet) lookup(name NormalizedName) *Flag { return f.formal[name] @@ -343,10 +394,11 @@ func (f *FlagSet) MarkDeprecated(name string, usageMessage string) error { if flag == nil { return fmt.Errorf("flag %q does not exist", name) } - if len(usageMessage) == 0 { + if usageMessage == "" { return fmt.Errorf("deprecated message for flag %q must be set", name) } flag.Deprecated = usageMessage + flag.Hidden = true return nil } @@ -358,7 +410,7 @@ func (f *FlagSet) MarkShorthandDeprecated(name string, usageMessage string) erro if flag == nil { return fmt.Errorf("flag %q does not exist", name) } - if len(usageMessage) == 0 { + if usageMessage == "" { return fmt.Errorf("deprecated message for flag %q must be set", name) } flag.ShorthandDeprecated = usageMessage @@ -382,6 +434,12 @@ func Lookup(name string) *Flag { return CommandLine.Lookup(name) } +// ShorthandLookup returns the Flag structure of the short handed flag, +// returning nil if none exists. +func ShorthandLookup(name string) *Flag { + return CommandLine.ShorthandLookup(name) +} + // Set sets the value of the named flag. func (f *FlagSet) Set(name, value string) error { normalName := f.normalizeFlagName(name) @@ -389,18 +447,30 @@ func (f *FlagSet) Set(name, value string) error { if !ok { return fmt.Errorf("no such flag -%v", name) } + err := flag.Value.Set(value) if err != nil { - return err + var flagName string + if flag.Shorthand != "" && flag.ShorthandDeprecated == "" { + flagName = fmt.Sprintf("-%s, --%s", flag.Shorthand, flag.Name) + } else { + flagName = fmt.Sprintf("--%s", flag.Name) + } + return fmt.Errorf("invalid argument %q for %q flag: %v", value, flagName, err) } - if f.actual == nil { - f.actual = make(map[NormalizedName]*Flag) + + if !flag.Changed { + if f.actual == nil { + f.actual = make(map[NormalizedName]*Flag) + } + f.actual[normalName] = flag + f.orderedActual = append(f.orderedActual, flag) + + flag.Changed = true } - f.actual[normalName] = flag - f.orderedActual = append(f.orderedActual, flag) - flag.Changed = true - if len(flag.Deprecated) > 0 { - fmt.Fprintf(os.Stderr, "Flag --%s has been deprecated, %s\n", flag.Name, flag.Deprecated) + + if flag.Deprecated != "" { + fmt.Fprintf(f.out(), "Flag --%s has been deprecated, %s\n", flag.Name, flag.Deprecated) } return nil } @@ -507,6 +577,14 @@ func UnquoteUsage(flag *Flag) (name string, usage string) { name = "int" case "uint64": name = "uint" + case "stringSlice": + name = "strings" + case "intSlice": + name = "ints" + case "uintSlice": + name = "uints" + case "boolSlice": + name = "bools" } return @@ -521,11 +599,14 @@ func wrapN(i, slop int, s string) (string, string) { return s, "" } - w := strings.LastIndexAny(s[:i], " \t") + w := strings.LastIndexAny(s[:i], " \t\n") if w <= 0 { return s, "" } - + nlPos := strings.LastIndex(s[:i], "\n") + if nlPos > 0 && nlPos < w { + return s[:nlPos], s[nlPos+1:] + } return s[:w], s[w+1:] } @@ -534,7 +615,7 @@ func wrapN(i, slop int, s string) (string, string) { // caller). Pass `w` == 0 to do no wrapping func wrap(i, w int, s string) string { if w == 0 { - return s + return strings.Replace(s, "\n", "\n"+strings.Repeat(" ", i), -1) } // space between indent i and end of line width w into which @@ -552,7 +633,7 @@ func wrap(i, w int, s string) string { } // If still not enough space then don't even try to wrap. if wrap < 24 { - return s + return strings.Replace(s, "\n", r, -1) } // Try to avoid short orphan words on the final line, by @@ -564,14 +645,14 @@ func wrap(i, w int, s string) string { // Handle first line, which is indented by the caller (or the // special case above) l, s = wrapN(wrap, slop, s) - r = r + l + r = r + strings.Replace(l, "\n", "\n"+strings.Repeat(" ", i), -1) // Now wrap the rest for s != "" { var t string t, s = wrapN(wrap, slop, s) - r = r + "\n" + strings.Repeat(" ", i) + t + r = r + "\n" + strings.Repeat(" ", i) + strings.Replace(t, "\n", "\n"+strings.Repeat(" ", i), -1) } return r @@ -582,28 +663,28 @@ func wrap(i, w int, s string) string { // for all flags in the FlagSet. Wrapped to `cols` columns (0 for no // wrapping) func (f *FlagSet) FlagUsagesWrapped(cols int) string { - x := new(bytes.Buffer) + buf := new(bytes.Buffer) lines := make([]string, 0, len(f.formal)) maxlen := 0 f.VisitAll(func(flag *Flag) { - if len(flag.Deprecated) > 0 || flag.Hidden { + if flag.Hidden { return } line := "" - if len(flag.Shorthand) > 0 && len(flag.ShorthandDeprecated) == 0 { + if flag.Shorthand != "" && flag.ShorthandDeprecated == "" { line = fmt.Sprintf(" -%s, --%s", flag.Shorthand, flag.Name) } else { line = fmt.Sprintf(" --%s", flag.Name) } varname, usage := UnquoteUsage(flag) - if len(varname) > 0 { + if varname != "" { line += " " + varname } - if len(flag.NoOptDefVal) > 0 { + if flag.NoOptDefVal != "" { switch flag.Value.Type() { case "string": line += fmt.Sprintf("[=\"%s\"]", flag.NoOptDefVal) @@ -611,6 +692,10 @@ func (f *FlagSet) FlagUsagesWrapped(cols int) string { if flag.NoOptDefVal != "true" { line += fmt.Sprintf("[=%s]", flag.NoOptDefVal) } + case "count": + if flag.NoOptDefVal != "+1" { + line += fmt.Sprintf("[=%s]", flag.NoOptDefVal) + } default: line += fmt.Sprintf("[=%s]", flag.NoOptDefVal) } @@ -631,6 +716,9 @@ func (f *FlagSet) FlagUsagesWrapped(cols int) string { line += fmt.Sprintf(" (default %s)", flag.DefValue) } } + if len(flag.Deprecated) != 0 { + line += fmt.Sprintf(" (DEPRECATED: %s)", flag.Deprecated) + } lines = append(lines, line) }) @@ -639,10 +727,10 @@ func (f *FlagSet) FlagUsagesWrapped(cols int) string { sidx := strings.Index(line, "\x00") spacing := strings.Repeat(" ", maxlen-sidx) // maxlen + 2 comes from + 1 for the \x00 and + 1 for the (deliberate) off-by-one in maxlen-sidx - fmt.Fprintln(x, line[:sidx], spacing, wrap(maxlen+2, cols, line[sidx+1:])) + fmt.Fprintln(buf, line[:sidx], spacing, wrap(maxlen+2, cols, line[sidx+1:])) } - return x.String() + return buf.String() } // FlagUsages returns a string containing the usage information for all flags in @@ -739,11 +827,10 @@ func (f *FlagSet) VarP(value Value, name, shorthand, usage string) { // AddFlag will add the flag to the FlagSet func (f *FlagSet) AddFlag(flag *Flag) { - // Call normalizeFlagName function only once normalizedFlagName := f.normalizeFlagName(flag.Name) - _, alreadythere := f.formal[normalizedFlagName] - if alreadythere { + _, alreadyThere := f.formal[normalizedFlagName] + if alreadyThere { msg := fmt.Sprintf("%s flag redefined: %s", f.name, flag.Name) fmt.Fprintln(f.out(), msg) panic(msg) // Happens only if flags are declared with identical names @@ -756,27 +843,29 @@ func (f *FlagSet) AddFlag(flag *Flag) { f.formal[normalizedFlagName] = flag f.orderedFormal = append(f.orderedFormal, flag) - if len(flag.Shorthand) == 0 { + if flag.Shorthand == "" { return } if len(flag.Shorthand) > 1 { - fmt.Fprintf(f.out(), "%s shorthand more than ASCII character: %s\n", f.name, flag.Shorthand) - panic("shorthand is more than one character") + msg := fmt.Sprintf("%q shorthand is more than one ASCII character", flag.Shorthand) + fmt.Fprintf(f.out(), msg) + panic(msg) } if f.shorthands == nil { f.shorthands = make(map[byte]*Flag) } c := flag.Shorthand[0] - old, alreadythere := f.shorthands[c] - if alreadythere { - fmt.Fprintf(f.out(), "%s shorthand reused: %q for %s already used for %s\n", f.name, c, flag.Name, old.Name) - panic("shorthand redefinition") + used, alreadyThere := f.shorthands[c] + if alreadyThere { + msg := fmt.Sprintf("unable to redefine %q shorthand in %q flagset: it's already used for %q flag", c, f.name, used.Name) + fmt.Fprintf(f.out(), msg) + panic(msg) } f.shorthands[c] = flag } // AddFlagSet adds one FlagSet to another. If a flag is already present in f -// the flag from newSet will be ignored +// the flag from newSet will be ignored. func (f *FlagSet) AddFlagSet(newSet *FlagSet) { if newSet == nil { return @@ -807,8 +896,10 @@ func VarP(value Value, name, shorthand, usage string) { // returns the error. func (f *FlagSet) failf(format string, a ...interface{}) error { err := fmt.Errorf(format, a...) - fmt.Fprintln(f.out(), err) - f.usage() + if f.errorHandling != ContinueOnError { + fmt.Fprintln(f.out(), err) + f.usage() + } return err } @@ -824,33 +915,23 @@ func (f *FlagSet) usage() { } } -func (f *FlagSet) setFlag(flag *Flag, value string, origArg string) error { - if err := flag.Value.Set(value); err != nil { - return f.failf("invalid argument %q for %s: %v", value, origArg, err) +//--unknown (args will be empty) +//--unknown --next-flag ... (args will be --next-flag ...) +//--unknown arg ... (args will be arg ...) +func stripUnknownFlagValue(args []string) []string { + if len(args) == 0 { + //--unknown + return args } - // mark as visited for Visit() - if f.actual == nil { - f.actual = make(map[NormalizedName]*Flag) - } - f.actual[f.normalizeFlagName(flag.Name)] = flag - f.orderedActual = append(f.orderedActual, flag) - flag.Changed = true - if len(flag.Deprecated) > 0 { - fmt.Fprintf(os.Stderr, "Flag --%s has been deprecated, %s\n", flag.Name, flag.Deprecated) - } - if len(flag.ShorthandDeprecated) > 0 && containsShorthand(origArg, flag.Shorthand) { - fmt.Fprintf(os.Stderr, "Flag shorthand -%s has been deprecated, %s\n", flag.Shorthand, flag.ShorthandDeprecated) - } - return nil -} -func containsShorthand(arg, shorthand string) bool { - // filter out flags -- - if strings.HasPrefix(arg, "-") { - return false + first := args[0] + if first[0] == '-' { + //--unknown --next-flag ... + return args } - arg = strings.SplitN(arg, "=", 2)[0] - return strings.Contains(arg, shorthand) + + //--unknown arg ... (args will be arg ...) + return args[1:] } func (f *FlagSet) parseLongArg(s string, args []string, fn parseFunc) (a []string, err error) { @@ -860,22 +941,35 @@ func (f *FlagSet) parseLongArg(s string, args []string, fn parseFunc) (a []strin err = f.failf("bad flag syntax: %s", s) return } + split := strings.SplitN(name, "=", 2) name = split[0] - flag, alreadythere := f.formal[f.normalizeFlagName(name)] - if !alreadythere { - if name == "help" { // special case for nice help message. + flag, exists := f.formal[f.normalizeFlagName(name)] + + if !exists { + switch { + case name == "help": f.usage() return a, ErrHelp + case f.ParseErrorsWhitelist.UnknownFlags: + // --unknown=unknownval arg ... + // we do not want to lose arg in this case + if len(split) >= 2 { + return a, nil + } + + return stripUnknownFlagValue(a), nil + default: + err = f.failf("unknown flag: --%s", name) + return } - err = f.failf("unknown flag: --%s", name) - return } + var value string if len(split) == 2 { // '--flag=arg' value = split[1] - } else if len(flag.NoOptDefVal) > 0 { + } else if flag.NoOptDefVal != "" { // '--flag' (arg was optional) value = flag.NoOptDefVal } else if len(a) > 0 { @@ -887,7 +981,11 @@ func (f *FlagSet) parseLongArg(s string, args []string, fn parseFunc) (a []strin err = f.failf("flag needs an argument: %s", s) return } - err = fn(flag, value, s) + + err = fn(flag, value) + if err != nil { + f.failf(err.Error()) + } return } @@ -895,38 +993,64 @@ func (f *FlagSet) parseSingleShortArg(shorthands string, args []string, fn parse if strings.HasPrefix(shorthands, "test.") { return } + outArgs = args outShorts = shorthands[1:] c := shorthands[0] - flag, alreadythere := f.shorthands[c] - if !alreadythere { - if c == 'h' { // special case for nice help message. + flag, exists := f.shorthands[c] + if !exists { + switch { + case c == 'h': f.usage() err = ErrHelp return + case f.ParseErrorsWhitelist.UnknownFlags: + // '-f=arg arg ...' + // we do not want to lose arg in this case + if len(shorthands) > 2 && shorthands[1] == '=' { + outShorts = "" + return + } + + outArgs = stripUnknownFlagValue(outArgs) + return + default: + err = f.failf("unknown shorthand flag: %q in -%s", c, shorthands) + return } - //TODO continue on error - err = f.failf("unknown shorthand flag: %q in -%s", c, shorthands) - return } + var value string if len(shorthands) > 2 && shorthands[1] == '=' { + // '-f=arg' value = shorthands[2:] outShorts = "" - } else if len(flag.NoOptDefVal) > 0 { + } else if flag.NoOptDefVal != "" { + // '-f' (arg was optional) value = flag.NoOptDefVal } else if len(shorthands) > 1 { + // '-farg' value = shorthands[1:] outShorts = "" } else if len(args) > 0 { + // '-f arg' value = args[0] outArgs = args[1:] } else { + // '-f' (arg was required) err = f.failf("flag needs an argument: %q in -%s", c, shorthands) return } - err = fn(flag, value, shorthands) + + if flag.ShorthandDeprecated != "" { + fmt.Fprintf(f.out(), "Flag shorthand -%s has been deprecated, %s\n", flag.Shorthand, flag.ShorthandDeprecated) + } + + err = fn(flag, value) + if err != nil { + f.failf(err.Error()) + } return } @@ -934,6 +1058,7 @@ func (f *FlagSet) parseShortArg(s string, args []string, fn parseFunc) (a []stri a = args shorthands := s[1:] + // "shorthands" can be a series of shorthand letters of flags (e.g. "-vvv"). for len(shorthands) > 0 { shorthands, a, err = f.parseSingleShortArg(shorthands, args, fn) if err != nil { @@ -980,19 +1105,30 @@ func (f *FlagSet) parseArgs(args []string, fn parseFunc) (err error) { // are defined and before flags are accessed by the program. // The return value will be ErrHelp if -help was set but not defined. func (f *FlagSet) Parse(arguments []string) error { + if f.addedGoFlagSets != nil { + for _, goFlagSet := range f.addedGoFlagSets { + goFlagSet.Parse(nil) + } + } f.parsed = true - f.args = make([]string, 0, len(arguments)) - assign := func(flag *Flag, value, origArg string) error { - return f.setFlag(flag, value, origArg) + if len(arguments) < 0 { + return nil } - err := f.parseArgs(arguments, assign) + f.args = make([]string, 0, len(arguments)) + + set := func(flag *Flag, value string) error { + return f.Set(flag.Name, value) + } + + err := f.parseArgs(arguments, set) if err != nil { switch f.errorHandling { case ContinueOnError: return err case ExitOnError: + fmt.Println(err) os.Exit(2) case PanicOnError: panic(err) @@ -1001,7 +1137,7 @@ func (f *FlagSet) Parse(arguments []string) error { return nil } -type parseFunc func(flag *Flag, value, origArg string) error +type parseFunc func(flag *Flag, value string) error // ParseAll parses flag definitions from the argument list, which should not // include the command name. The arguments for fn are flag and value. Must be @@ -1012,11 +1148,7 @@ func (f *FlagSet) ParseAll(arguments []string, fn func(flag *Flag, value string) f.parsed = true f.args = make([]string, 0, len(arguments)) - assign := func(flag *Flag, value, origArg string) error { - return fn(flag, value) - } - - err := f.parseArgs(arguments, assign) + err := f.parseArgs(arguments, fn) if err != nil { switch f.errorHandling { case ContinueOnError: diff --git a/vendor/github.com/spf13/pflag/golangflag.go b/vendor/github.com/spf13/pflag/golangflag.go index c4f47ebe..d3dd72b7 100644 --- a/vendor/github.com/spf13/pflag/golangflag.go +++ b/vendor/github.com/spf13/pflag/golangflag.go @@ -98,4 +98,8 @@ func (f *FlagSet) AddGoFlagSet(newSet *goflag.FlagSet) { newSet.VisitAll(func(goflag *goflag.Flag) { f.AddGoFlag(goflag) }) + if f.addedGoFlagSets == nil { + f.addedGoFlagSets = make([]*goflag.FlagSet, 0) + } + f.addedGoFlagSets = append(f.addedGoFlagSets, newSet) } diff --git a/vendor/github.com/spf13/pflag/int16.go b/vendor/github.com/spf13/pflag/int16.go new file mode 100644 index 00000000..f1a01d05 --- /dev/null +++ b/vendor/github.com/spf13/pflag/int16.go @@ -0,0 +1,88 @@ +package pflag + +import "strconv" + +// -- int16 Value +type int16Value int16 + +func newInt16Value(val int16, p *int16) *int16Value { + *p = val + return (*int16Value)(p) +} + +func (i *int16Value) Set(s string) error { + v, err := strconv.ParseInt(s, 0, 16) + *i = int16Value(v) + return err +} + +func (i *int16Value) Type() string { + return "int16" +} + +func (i *int16Value) String() string { return strconv.FormatInt(int64(*i), 10) } + +func int16Conv(sval string) (interface{}, error) { + v, err := strconv.ParseInt(sval, 0, 16) + if err != nil { + return 0, err + } + return int16(v), nil +} + +// GetInt16 returns the int16 value of a flag with the given name +func (f *FlagSet) GetInt16(name string) (int16, error) { + val, err := f.getFlagType(name, "int16", int16Conv) + if err != nil { + return 0, err + } + return val.(int16), nil +} + +// Int16Var defines an int16 flag with specified name, default value, and usage string. +// The argument p points to an int16 variable in which to store the value of the flag. +func (f *FlagSet) Int16Var(p *int16, name string, value int16, usage string) { + f.VarP(newInt16Value(value, p), name, "", usage) +} + +// Int16VarP is like Int16Var, but accepts a shorthand letter that can be used after a single dash. +func (f *FlagSet) Int16VarP(p *int16, name, shorthand string, value int16, usage string) { + f.VarP(newInt16Value(value, p), name, shorthand, usage) +} + +// Int16Var defines an int16 flag with specified name, default value, and usage string. +// The argument p points to an int16 variable in which to store the value of the flag. +func Int16Var(p *int16, name string, value int16, usage string) { + CommandLine.VarP(newInt16Value(value, p), name, "", usage) +} + +// Int16VarP is like Int16Var, but accepts a shorthand letter that can be used after a single dash. +func Int16VarP(p *int16, name, shorthand string, value int16, usage string) { + CommandLine.VarP(newInt16Value(value, p), name, shorthand, usage) +} + +// Int16 defines an int16 flag with specified name, default value, and usage string. +// The return value is the address of an int16 variable that stores the value of the flag. +func (f *FlagSet) Int16(name string, value int16, usage string) *int16 { + p := new(int16) + f.Int16VarP(p, name, "", value, usage) + return p +} + +// Int16P is like Int16, but accepts a shorthand letter that can be used after a single dash. +func (f *FlagSet) Int16P(name, shorthand string, value int16, usage string) *int16 { + p := new(int16) + f.Int16VarP(p, name, shorthand, value, usage) + return p +} + +// Int16 defines an int16 flag with specified name, default value, and usage string. +// The return value is the address of an int16 variable that stores the value of the flag. +func Int16(name string, value int16, usage string) *int16 { + return CommandLine.Int16P(name, "", value, usage) +} + +// Int16P is like Int16, but accepts a shorthand letter that can be used after a single dash. +func Int16P(name, shorthand string, value int16, usage string) *int16 { + return CommandLine.Int16P(name, shorthand, value, usage) +} diff --git a/vendor/github.com/spf13/pflag/string_array.go b/vendor/github.com/spf13/pflag/string_array.go index 276b7ed4..fa7bc601 100644 --- a/vendor/github.com/spf13/pflag/string_array.go +++ b/vendor/github.com/spf13/pflag/string_array.go @@ -52,7 +52,7 @@ func (f *FlagSet) GetStringArray(name string) ([]string, error) { // StringArrayVar defines a string flag with specified name, default value, and usage string. // The argument p points to a []string variable in which to store the values of the multiple flags. -// The value of each argument will not try to be separated by comma +// The value of each argument will not try to be separated by comma. Use a StringSlice for that. func (f *FlagSet) StringArrayVar(p *[]string, name string, value []string, usage string) { f.VarP(newStringArrayValue(value, p), name, "", usage) } @@ -64,7 +64,7 @@ func (f *FlagSet) StringArrayVarP(p *[]string, name, shorthand string, value []s // StringArrayVar defines a string flag with specified name, default value, and usage string. // The argument p points to a []string variable in which to store the value of the flag. -// The value of each argument will not try to be separated by comma +// The value of each argument will not try to be separated by comma. Use a StringSlice for that. func StringArrayVar(p *[]string, name string, value []string, usage string) { CommandLine.VarP(newStringArrayValue(value, p), name, "", usage) } @@ -76,7 +76,7 @@ func StringArrayVarP(p *[]string, name, shorthand string, value []string, usage // StringArray defines a string flag with specified name, default value, and usage string. // The return value is the address of a []string variable that stores the value of the flag. -// The value of each argument will not try to be separated by comma +// The value of each argument will not try to be separated by comma. Use a StringSlice for that. func (f *FlagSet) StringArray(name string, value []string, usage string) *[]string { p := []string{} f.StringArrayVarP(&p, name, "", value, usage) @@ -92,7 +92,7 @@ func (f *FlagSet) StringArrayP(name, shorthand string, value []string, usage str // StringArray defines a string flag with specified name, default value, and usage string. // The return value is the address of a []string variable that stores the value of the flag. -// The value of each argument will not try to be separated by comma +// The value of each argument will not try to be separated by comma. Use a StringSlice for that. func StringArray(name string, value []string, usage string) *[]string { return CommandLine.StringArrayP(name, "", value, usage) } diff --git a/vendor/github.com/spf13/pflag/string_slice.go b/vendor/github.com/spf13/pflag/string_slice.go index 05eee754..0cd3ccc0 100644 --- a/vendor/github.com/spf13/pflag/string_slice.go +++ b/vendor/github.com/spf13/pflag/string_slice.go @@ -82,6 +82,11 @@ func (f *FlagSet) GetStringSlice(name string) ([]string, error) { // StringSliceVar defines a string flag with specified name, default value, and usage string. // The argument p points to a []string variable in which to store the value of the flag. +// Compared to StringArray flags, StringSlice flags take comma-separated value as arguments and split them accordingly. +// For example: +// --ss="v1,v2" -ss="v3" +// will result in +// []string{"v1", "v2", "v3"} func (f *FlagSet) StringSliceVar(p *[]string, name string, value []string, usage string) { f.VarP(newStringSliceValue(value, p), name, "", usage) } @@ -93,6 +98,11 @@ func (f *FlagSet) StringSliceVarP(p *[]string, name, shorthand string, value []s // StringSliceVar defines a string flag with specified name, default value, and usage string. // The argument p points to a []string variable in which to store the value of the flag. +// Compared to StringArray flags, StringSlice flags take comma-separated value as arguments and split them accordingly. +// For example: +// --ss="v1,v2" -ss="v3" +// will result in +// []string{"v1", "v2", "v3"} func StringSliceVar(p *[]string, name string, value []string, usage string) { CommandLine.VarP(newStringSliceValue(value, p), name, "", usage) } @@ -104,6 +114,11 @@ func StringSliceVarP(p *[]string, name, shorthand string, value []string, usage // StringSlice defines a string flag with specified name, default value, and usage string. // The return value is the address of a []string variable that stores the value of the flag. +// Compared to StringArray flags, StringSlice flags take comma-separated value as arguments and split them accordingly. +// For example: +// --ss="v1,v2" -ss="v3" +// will result in +// []string{"v1", "v2", "v3"} func (f *FlagSet) StringSlice(name string, value []string, usage string) *[]string { p := []string{} f.StringSliceVarP(&p, name, "", value, usage) @@ -119,6 +134,11 @@ func (f *FlagSet) StringSliceP(name, shorthand string, value []string, usage str // StringSlice defines a string flag with specified name, default value, and usage string. // The return value is the address of a []string variable that stores the value of the flag. +// Compared to StringArray flags, StringSlice flags take comma-separated value as arguments and split them accordingly. +// For example: +// --ss="v1,v2" -ss="v3" +// will result in +// []string{"v1", "v2", "v3"} func StringSlice(name string, value []string, usage string) *[]string { return CommandLine.StringSliceP(name, "", value, usage) } diff --git a/vendor/github.com/spf13/viper/README.md b/vendor/github.com/spf13/viper/README.md index 25181dff..64bf4743 100644 --- a/vendor/github.com/spf13/viper/README.md +++ b/vendor/github.com/spf13/viper/README.md @@ -6,18 +6,19 @@ Many Go projects are built using Viper including: * [Hugo](http://gohugo.io) * [EMC RexRay](http://rexray.readthedocs.org/en/stable/) -* [Imgur's Incus](https://github.com/Imgur/incus) +* [Imgur’s Incus](https://github.com/Imgur/incus) * [Nanobox](https://github.com/nanobox-io/nanobox)/[Nanopack](https://github.com/nanopack) * [Docker Notary](https://github.com/docker/Notary) * [BloomApi](https://www.bloomapi.com/) * [doctl](https://github.com/digitalocean/doctl) +* [Clairctl](https://github.com/jgsqware/clairctl) [![Build Status](https://travis-ci.org/spf13/viper.svg)](https://travis-ci.org/spf13/viper) [![Join the chat at https://gitter.im/spf13/viper](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/spf13/viper?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![GoDoc](https://godoc.org/github.com/spf13/viper?status.svg)](https://godoc.org/github.com/spf13/viper) ## What is Viper? -Viper is a complete configuration solution for go applications including 12 factor apps. It is designed +Viper is a complete configuration solution for Go applications including 12-Factor apps. It is designed to work within an application, and can handle all types of configuration needs and formats. It supports: @@ -68,7 +69,7 @@ Viper configuration keys are case insensitive. ### Establishing Defaults A good configuration system will support default values. A default value is not -required for a key, but it's useful in the event that a key hasn’t been set via +required for a key, but it’s useful in the event that a key hasn’t been set via config file, environment variable, remote configuration or flag. Examples: @@ -116,10 +117,10 @@ Optionally you can provide a function for Viper to run each time a change occurs **Make sure you add all of the configPaths prior to calling `WatchConfig()`** ```go - viper.WatchConfig() - viper.OnConfigChange(func(e fsnotify.Event) { - fmt.Println("Config file changed:", e.Name) - }) +viper.WatchConfig() +viper.OnConfigChange(func(e fsnotify.Event) { + fmt.Println("Config file changed:", e.Name) +}) ``` ### Reading Config from io.Reader @@ -184,7 +185,7 @@ with ENV: * `AutomaticEnv()` * `BindEnv(string...) : error` * `SetEnvPrefix(string)` - * `SetEnvReplacer(string...) *strings.Replacer` + * `SetEnvKeyReplacer(string...) *strings.Replacer` _When working with ENV variables, it’s important to recognize that Viper treats ENV variables as case sensitive._ @@ -211,7 +212,7 @@ time a `viper.Get` request is made. It will apply the following rules. It will check for a environment variable with a name matching the key uppercased and prefixed with the `EnvPrefix` if set. -`SetEnvReplacer` allows you to use a `strings.Replacer` object to rewrite Env +`SetEnvKeyReplacer` allows you to use a `strings.Replacer` object to rewrite Env keys to an extent. This is useful if you want to use `-` or something in your `Get()` calls, but want your environmental variables to use `_` delimiters. An example of using it can be found in `viper_test.go`. @@ -236,7 +237,7 @@ Like `BindEnv`, the value is not set when the binding method is called, but when it is accessed. This means you can bind as early as you want, even in an `init()` function. -The `BindPFlag()` method provides this functionality. +For individual flags, the `BindPFlag()` method provides this functionality. Example: @@ -245,6 +246,19 @@ serverCmd.Flags().Int("port", 1138, "Port to run Application server on") viper.BindPFlag("port", serverCmd.Flags().Lookup("port")) ``` +You can also bind an existing set of pflags (pflag.FlagSet): + +Example: + +```go +pflag.Int("flagname", 1234, "help message for flagname") + +pflag.Parse() +viper.BindPFlags(pflag.CommandLine) + +i := viper.GetInt("flagname") // retrieve values from viper instead of pflag +``` + The use of [pflag](https://github.com/spf13/pflag/) in Viper does not preclude the use of other packages that use the [flag](https://golang.org/pkg/flag/) package from the standard library. The pflag package can handle the flags @@ -263,15 +277,23 @@ import ( ) func main() { + + // using standard library "flag" package + flag.Int("flagname", 1234, "help message for flagname") + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.Parse() - ... + viper.BindPFlags(pflag.CommandLine) + + i := viper.GetInt("flagname") // retrieve value from viper + + ... } ``` #### Flag interfaces -Viper provides two Go interfaces to bind other flag systems if you don't use `Pflags`. +Viper provides two Go interfaces to bind other flag systems if you don’t use `Pflags`. `FlagValue` represents a single flag. This is a very simple example on how to implement this interface: @@ -401,7 +423,7 @@ go func(){ ## Getting Values From Viper -In Viper, there are a few ways to get a value depending on the value's type. +In Viper, there are a few ways to get a value depending on the value’s type. The following functions and methods exist: * `Get(key string) : interface{}` @@ -531,7 +553,7 @@ func NewCache(cfg *Viper) *Cache {...} ``` which creates a cache based on config information formatted as `subv`. -Now it's easy to create these 2 caches separately as: +Now it’s easy to create these 2 caches separately as: ```go cfg1 := viper.Sub("app.cache1") @@ -575,13 +597,13 @@ initialization needed to begin using Viper. Since most applications will want to use a single central repository for their configuration, the viper package provides this. It is similar to a singleton. -In all of the examples above, they demonstrate using viper in it's singleton +In all of the examples above, they demonstrate using viper in its singleton style approach. ### Working with multiple vipers You can also create many different vipers for use in your application. Each will -have it’s own unique set of configurations and values. Each can read from a +have its own unique set of configurations and values. Each can read from a different config file, key value store, etc. All of the functions that viper package supports are mirrored as methods on a viper. diff --git a/vendor/github.com/spf13/viper/nohup.out b/vendor/github.com/spf13/viper/nohup.out deleted file mode 100644 index 8973bf27..00000000 --- a/vendor/github.com/spf13/viper/nohup.out +++ /dev/null @@ -1 +0,0 @@ -QProcess::start: Process is already running diff --git a/vendor/github.com/spf13/viper/util.go b/vendor/github.com/spf13/viper/util.go index 3ebada91..952cad44 100644 --- a/vendor/github.com/spf13/viper/util.go +++ b/vendor/github.com/spf13/viper/util.go @@ -11,22 +11,16 @@ package viper import ( - "bytes" - "encoding/json" "fmt" - "io" "os" "path/filepath" "runtime" "strings" "unicode" - "github.com/hashicorp/hcl" - "github.com/magiconair/properties" - toml "github.com/pelletier/go-toml" + "github.com/spf13/afero" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" - "gopkg.in/yaml.v2" ) // ConfigParseError denotes failing to parse configuration file. @@ -121,8 +115,8 @@ func absPathify(inPath string) string { } // Check if File / Directory Exists -func exists(path string) (bool, error) { - _, err := v.fs.Stat(path) +func exists(fs afero.Fs, path string) (bool, error) { + _, err := fs.Stat(path) if err == nil { return true, nil } @@ -152,61 +146,6 @@ func userHomeDir() string { return os.Getenv("HOME") } -func unmarshallConfigReader(in io.Reader, c map[string]interface{}, configType string) error { - buf := new(bytes.Buffer) - buf.ReadFrom(in) - - switch strings.ToLower(configType) { - case "yaml", "yml": - if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil { - return ConfigParseError{err} - } - - case "json": - if err := json.Unmarshal(buf.Bytes(), &c); err != nil { - return ConfigParseError{err} - } - - case "hcl": - obj, err := hcl.Parse(string(buf.Bytes())) - if err != nil { - return ConfigParseError{err} - } - if err = hcl.DecodeObject(&c, obj); err != nil { - return ConfigParseError{err} - } - - case "toml": - tree, err := toml.LoadReader(buf) - if err != nil { - return ConfigParseError{err} - } - tmap := tree.ToMap() - for k, v := range tmap { - c[k] = v - } - - case "properties", "props", "prop": - var p *properties.Properties - var err error - if p, err = properties.Load(buf.Bytes(), properties.UTF8); err != nil { - return ConfigParseError{err} - } - for _, key := range p.Keys() { - value, _ := p.Get(key) - // recursively build nested maps - path := strings.Split(key, ".") - lastKey := strings.ToLower(path[len(path)-1]) - deepestMap := deepSearch(c, path[0:len(path)-1]) - // set innermost value - deepestMap[lastKey] = value - } - } - - insensitiviseMap(c) - return nil -} - func safeMul(a, b uint) uint { c := a * b if a > 1 && b > 1 && c/b != a { diff --git a/vendor/github.com/spf13/viper/viper.go b/vendor/github.com/spf13/viper/viper.go index 31b41a6b..e9966ba5 100644 --- a/vendor/github.com/spf13/viper/viper.go +++ b/vendor/github.com/spf13/viper/viper.go @@ -22,6 +22,7 @@ package viper import ( "bytes" "encoding/csv" + "encoding/json" "fmt" "io" "log" @@ -31,14 +32,30 @@ import ( "strings" "time" + yaml "gopkg.in/yaml.v2" + "github.com/fsnotify/fsnotify" + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/printer" + "github.com/magiconair/properties" "github.com/mitchellh/mapstructure" + toml "github.com/pelletier/go-toml" "github.com/spf13/afero" "github.com/spf13/cast" jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" ) +// ConfigMarshalError happens when failing to marshal the configuration. +type ConfigMarshalError struct { + err error +} + +// Error returns the formatted configuration error. +func (e ConfigMarshalError) Error() string { + return fmt.Sprintf("While marshaling config: %s", e.err.Error()) +} + var v *Viper type RemoteResponse struct { @@ -53,7 +70,7 @@ func init() { type remoteConfigFactory interface { Get(rp RemoteProvider) (io.Reader, error) Watch(rp RemoteProvider) (io.Reader, error) - WatchChannel(rp RemoteProvider)(<-chan *RemoteResponse, chan bool) + WatchChannel(rp RemoteProvider) (<-chan *RemoteResponse, chan bool) } // RemoteConfig is optional, see the remote package @@ -69,8 +86,7 @@ func (str UnsupportedConfigError) Error() string { } // UnsupportedRemoteProviderError denotes encountering an unsupported remote -// provider. Currently only etcd and Consul are -// supported. +// provider. Currently only etcd and Consul are supported. type UnsupportedRemoteProviderError string // Error returns the formatted remote provider error. @@ -163,6 +179,10 @@ type Viper struct { aliases map[string]string typeByDefValue bool + // Store read properties on the object so that we can write back in order with comments. + // This will only be used if the configuration read is a properties file. + properties *properties.Properties + onConfigChange func(fsnotify.Event) } @@ -189,7 +209,7 @@ func New() *Viper { // can use it in their testing as well. func Reset() { v = New() - SupportedExts = []string{"json", "toml", "yaml", "yml", "hcl"} + SupportedExts = []string{"json", "toml", "yaml", "yml", "properties", "props", "prop", "hcl"} SupportedRemoteProviders = []string{"etcd", "consul"} } @@ -283,8 +303,8 @@ func (v *Viper) WatchConfig() { }() } -// SetConfigFile explicitly defines the path, name and extension of the config file -// Viper will use this and not check any of the config paths +// SetConfigFile explicitly defines the path, name and extension of the config file. +// Viper will use this and not check any of the config paths. func SetConfigFile(in string) { v.SetConfigFile(in) } func (v *Viper) SetConfigFile(in string) { if in != "" { @@ -293,8 +313,8 @@ func (v *Viper) SetConfigFile(in string) { } // SetEnvPrefix defines a prefix that ENVIRONMENT variables will use. -// E.g. if your prefix is "spf", the env registry -// will look for env. variables that start with "SPF_" +// E.g. if your prefix is "spf", the env registry will look for env +// variables that start with "SPF_". func SetEnvPrefix(in string) { v.SetEnvPrefix(in) } func (v *Viper) SetEnvPrefix(in string) { if in != "" { @@ -312,11 +332,11 @@ func (v *Viper) mergeWithEnvPrefix(in string) string { // TODO: should getEnv logic be moved into find(). Can generalize the use of // rewriting keys many things, Ex: Get('someKey') -> some_key -// (cammel case to snake case for JSON keys perhaps) +// (camel case to snake case for JSON keys perhaps) // getEnv is a wrapper around os.Getenv which replaces characters in the original -// key. This allows env vars which have different keys then the config object -// keys +// key. This allows env vars which have different keys than the config object +// keys. func (v *Viper) getEnv(key string) string { if v.envKeyReplacer != nil { key = v.envKeyReplacer.Replace(key) @@ -324,7 +344,7 @@ func (v *Viper) getEnv(key string) string { return os.Getenv(key) } -// ConfigFileUsed returns the file used to populate the config registry +// ConfigFileUsed returns the file used to populate the config registry. func ConfigFileUsed() string { return v.ConfigFileUsed() } func (v *Viper) ConfigFileUsed() string { return v.configFile } @@ -597,32 +617,33 @@ func (v *Viper) Get(key string) interface{} { return nil } - valType := val if v.typeByDefValue { // TODO(bep) this branch isn't covered by a single test. + valType := val path := strings.Split(lcaseKey, v.keyDelim) defVal := v.searchMap(v.defaults, path) if defVal != nil { valType = defVal } + + switch valType.(type) { + case bool: + return cast.ToBool(val) + case string: + return cast.ToString(val) + case int64, int32, int16, int8, int: + return cast.ToInt(val) + case float64, float32: + return cast.ToFloat64(val) + case time.Time: + return cast.ToTime(val) + case time.Duration: + return cast.ToDuration(val) + case []string: + return cast.ToStringSlice(val) + } } - switch valType.(type) { - case bool: - return cast.ToBool(val) - case string: - return cast.ToString(val) - case int64, int32, int16, int8, int: - return cast.ToInt(val) - case float64, float32: - return cast.ToFloat64(val) - case time.Time: - return cast.ToTime(val) - case time.Duration: - return cast.ToDuration(val) - case []string: - return cast.ToStringSlice(val) - } return val } @@ -747,13 +768,16 @@ func (v *Viper) Unmarshal(rawVal interface{}) error { } // defaultDecoderConfig returns default mapsstructure.DecoderConfig with suppot -// of time.Duration values +// of time.Duration values & string slices func defaultDecoderConfig(output interface{}) *mapstructure.DecoderConfig { return &mapstructure.DecoderConfig{ Metadata: nil, Result: output, WeaklyTypedInput: true, - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + DecodeHook: mapstructure.ComposeDecodeHookFunc( + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + ), } } @@ -814,7 +838,7 @@ func (v *Viper) BindFlagValues(flags FlagValueSet) (err error) { } // BindFlagValue binds a specific key to a FlagValue. -// Example(where serverCmd is a Cobra instance): +// Example (where serverCmd is a Cobra instance): // // serverCmd.Flags().Int("port", 1138, "Port to run Application server on") // Viper.BindFlagValue("port", serverCmd.Flags().Lookup("port")) @@ -1116,6 +1140,7 @@ func (v *Viper) ReadInConfig() error { return UnsupportedConfigError(v.getConfigType()) } + jww.DEBUG.Println("Reading file: ", filename) file, err := afero.ReadFile(v.fs, filename) if err != nil { return err @@ -1175,6 +1200,195 @@ func (v *Viper) MergeConfig(in io.Reader) error { return nil } +// WriteConfig writes the current configuration to a file. +func WriteConfig() error { return v.WriteConfig() } +func (v *Viper) WriteConfig() error { + filename, err := v.getConfigFile() + if err != nil { + return err + } + return v.writeConfig(filename, true) +} + +// SafeWriteConfig writes current configuration to file only if the file does not exist. +func SafeWriteConfig() error { return v.SafeWriteConfig() } +func (v *Viper) SafeWriteConfig() error { + filename, err := v.getConfigFile() + if err != nil { + return err + } + return v.writeConfig(filename, false) +} + +// WriteConfigAs writes current configuration to a given filename. +func WriteConfigAs(filename string) error { return v.WriteConfigAs(filename) } +func (v *Viper) WriteConfigAs(filename string) error { + return v.writeConfig(filename, true) +} + +// SafeWriteConfigAs writes current configuration to a given filename if it does not exist. +func SafeWriteConfigAs(filename string) error { return v.SafeWriteConfigAs(filename) } +func (v *Viper) SafeWriteConfigAs(filename string) error { + return v.writeConfig(filename, false) +} + +func writeConfig(filename string, force bool) error { return v.writeConfig(filename, force) } +func (v *Viper) writeConfig(filename string, force bool) error { + jww.INFO.Println("Attempting to write configuration to file.") + ext := filepath.Ext(filename) + if len(ext) <= 1 { + return fmt.Errorf("Filename: %s requires valid extension.", filename) + } + configType := ext[1:] + if !stringInSlice(configType, SupportedExts) { + return UnsupportedConfigError(configType) + } + if v.config == nil { + v.config = make(map[string]interface{}) + } + var flags int + if force == true { + flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY + } else { + if _, err := os.Stat(filename); os.IsNotExist(err) { + flags = os.O_WRONLY + } else { + return fmt.Errorf("File: %s exists. Use WriteConfig to overwrite.", filename) + } + } + f, err := v.fs.OpenFile(filename, flags, os.FileMode(0644)) + if err != nil { + return err + } + return v.marshalWriter(f, configType) +} + +// Unmarshal a Reader into a map. +// Should probably be an unexported function. +func unmarshalReader(in io.Reader, c map[string]interface{}) error { + return v.unmarshalReader(in, c) +} +func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { + buf := new(bytes.Buffer) + buf.ReadFrom(in) + + switch strings.ToLower(v.getConfigType()) { + case "yaml", "yml": + if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil { + return ConfigParseError{err} + } + + case "json": + if err := json.Unmarshal(buf.Bytes(), &c); err != nil { + return ConfigParseError{err} + } + + case "hcl": + obj, err := hcl.Parse(string(buf.Bytes())) + if err != nil { + return ConfigParseError{err} + } + if err = hcl.DecodeObject(&c, obj); err != nil { + return ConfigParseError{err} + } + + case "toml": + tree, err := toml.LoadReader(buf) + if err != nil { + return ConfigParseError{err} + } + tmap := tree.ToMap() + for k, v := range tmap { + c[k] = v + } + + case "properties", "props", "prop": + v.properties = properties.NewProperties() + var err error + if v.properties, err = properties.Load(buf.Bytes(), properties.UTF8); err != nil { + return ConfigParseError{err} + } + for _, key := range v.properties.Keys() { + value, _ := v.properties.Get(key) + // recursively build nested maps + path := strings.Split(key, ".") + lastKey := strings.ToLower(path[len(path)-1]) + deepestMap := deepSearch(c, path[0:len(path)-1]) + // set innermost value + deepestMap[lastKey] = value + } + } + + insensitiviseMap(c) + return nil +} + +// Marshal a map into Writer. +func marshalWriter(f afero.File, configType string) error { + return v.marshalWriter(f, configType) +} +func (v *Viper) marshalWriter(f afero.File, configType string) error { + c := v.AllSettings() + switch configType { + case "json": + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + return ConfigMarshalError{err} + } + _, err = f.WriteString(string(b)) + if err != nil { + return ConfigMarshalError{err} + } + + case "hcl": + b, err := json.Marshal(c) + ast, err := hcl.Parse(string(b)) + if err != nil { + return ConfigMarshalError{err} + } + err = printer.Fprint(f, ast.Node) + if err != nil { + return ConfigMarshalError{err} + } + + case "prop", "props", "properties": + if v.properties == nil { + v.properties = properties.NewProperties() + } + p := v.properties + for _, key := range v.AllKeys() { + _, _, err := p.Set(key, v.GetString(key)) + if err != nil { + return ConfigMarshalError{err} + } + } + _, err := p.WriteComment(f, "#", properties.UTF8) + if err != nil { + return ConfigMarshalError{err} + } + + case "toml": + t, err := toml.TreeFromMap(c) + if err != nil { + return ConfigMarshalError{err} + } + s := t.String() + if _, err := f.WriteString(s); err != nil { + return ConfigMarshalError{err} + } + + case "yaml", "yml": + b, err := yaml.Marshal(c) + if err != nil { + return ConfigMarshalError{err} + } + if _, err = f.WriteString(string(b)); err != nil { + return ConfigMarshalError{err} + } + } + return nil +} + func keyExists(k string, m map[string]interface{}) string { lk := strings.ToLower(k) for mk := range m { @@ -1287,16 +1501,6 @@ func (v *Viper) WatchRemoteConfigOnChannel() error { return v.watchKeyValueConfigOnChannel() } -// Unmarshall a Reader into a map. -// Should probably be an unexported function. -func unmarshalReader(in io.Reader, c map[string]interface{}) error { - return v.unmarshalReader(in, c) -} - -func (v *Viper) unmarshalReader(in io.Reader, c map[string]interface{}) error { - return unmarshallConfigReader(in, c, v.getConfigType()) -} - func (v *Viper) insensitiviseMaps() { insensitiviseMap(v.config) insensitiviseMap(v.defaults) @@ -1516,25 +1720,21 @@ func (v *Viper) getConfigType() string { } func (v *Viper) getConfigFile() (string, error) { - // if explicitly set, then use it - if v.configFile != "" { - return v.configFile, nil + if v.configFile == "" { + cf, err := v.findConfigFile() + if err != nil { + return "", err + } + v.configFile = cf } - - cf, err := v.findConfigFile() - if err != nil { - return "", err - } - - v.configFile = cf - return v.getConfigFile() + return v.configFile, nil } func (v *Viper) searchInPath(in string) (filename string) { jww.DEBUG.Println("Searching for config in ", in) for _, ext := range SupportedExts { jww.DEBUG.Println("Checking for", filepath.Join(in, v.configName+"."+ext)) - if b, _ := exists(filepath.Join(in, v.configName+"."+ext)); b { + if b, _ := exists(v.fs, filepath.Join(in, v.configName+"."+ext)); b { jww.DEBUG.Println("Found: ", filepath.Join(in, v.configName+"."+ext)) return filepath.Join(in, v.configName+"."+ext) } @@ -1546,7 +1746,6 @@ func (v *Viper) searchInPath(in string) (filename string) { // Search all configPaths for any config file. // Returns the first path that exists (and is a config file). func (v *Viper) findConfigFile() (string, error) { - jww.INFO.Println("Searching for config in ", v.configPaths) for _, cp := range v.configPaths { diff --git a/vendor/github.com/tinylib/msgp/LICENSE b/vendor/github.com/tinylib/msgp/LICENSE new file mode 100644 index 00000000..14d60424 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/LICENSE @@ -0,0 +1,8 @@ +Copyright (c) 2014 Philip Hofer +Portions Copyright (c) 2009 The Go Authors (license at http://golang.org) where indicated + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/tinylib/msgp/msgp/advise_linux.go b/vendor/github.com/tinylib/msgp/msgp/advise_linux.go new file mode 100644 index 00000000..6c6bb37a --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/advise_linux.go @@ -0,0 +1,24 @@ +// +build linux,!appengine + +package msgp + +import ( + "os" + "syscall" +) + +func adviseRead(mem []byte) { + syscall.Madvise(mem, syscall.MADV_SEQUENTIAL|syscall.MADV_WILLNEED) +} + +func adviseWrite(mem []byte) { + syscall.Madvise(mem, syscall.MADV_SEQUENTIAL) +} + +func fallocate(f *os.File, sz int64) error { + err := syscall.Fallocate(int(f.Fd()), 0, 0, sz) + if err == syscall.ENOTSUP { + return f.Truncate(sz) + } + return err +} diff --git a/vendor/github.com/tinylib/msgp/msgp/advise_other.go b/vendor/github.com/tinylib/msgp/msgp/advise_other.go new file mode 100644 index 00000000..da65ea54 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/advise_other.go @@ -0,0 +1,17 @@ +// +build !linux appengine + +package msgp + +import ( + "os" +) + +// TODO: darwin, BSD support + +func adviseRead(mem []byte) {} + +func adviseWrite(mem []byte) {} + +func fallocate(f *os.File, sz int64) error { + return f.Truncate(sz) +} diff --git a/vendor/github.com/tinylib/msgp/msgp/circular.go b/vendor/github.com/tinylib/msgp/msgp/circular.go new file mode 100644 index 00000000..a0434c7e --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/circular.go @@ -0,0 +1,39 @@ +package msgp + +type timer interface { + StartTimer() + StopTimer() +} + +// EndlessReader is an io.Reader +// that loops over the same data +// endlessly. It is used for benchmarking. +type EndlessReader struct { + tb timer + data []byte + offset int +} + +// NewEndlessReader returns a new endless reader +func NewEndlessReader(b []byte, tb timer) *EndlessReader { + return &EndlessReader{tb: tb, data: b, offset: 0} +} + +// Read implements io.Reader. In practice, it +// always returns (len(p), nil), although it +// fills the supplied slice while the benchmark +// timer is stopped. +func (c *EndlessReader) Read(p []byte) (int, error) { + c.tb.StopTimer() + var n int + l := len(p) + m := len(c.data) + for n < l { + nn := copy(p[n:], c.data[c.offset:]) + n += nn + c.offset += nn + c.offset %= m + } + c.tb.StartTimer() + return n, nil +} diff --git a/vendor/github.com/tinylib/msgp/msgp/defs.go b/vendor/github.com/tinylib/msgp/msgp/defs.go new file mode 100644 index 00000000..c634eef1 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/defs.go @@ -0,0 +1,142 @@ +// This package is the support library for the msgp code generator (http://github.com/tinylib/msgp). +// +// This package defines the utilites used by the msgp code generator for encoding and decoding MessagePack +// from []byte and io.Reader/io.Writer types. Much of this package is devoted to helping the msgp code +// generator implement the Marshaler/Unmarshaler and Encodable/Decodable interfaces. +// +// This package defines four "families" of functions: +// - AppendXxxx() appends an object to a []byte in MessagePack encoding. +// - ReadXxxxBytes() reads an object from a []byte and returns the remaining bytes. +// - (*Writer).WriteXxxx() writes an object to the buffered *Writer type. +// - (*Reader).ReadXxxx() reads an object from a buffered *Reader type. +// +// Once a type has satisfied the `Encodable` and `Decodable` interfaces, +// it can be written and read from arbitrary `io.Writer`s and `io.Reader`s using +// msgp.Encode(io.Writer, msgp.Encodable) +// and +// msgp.Decode(io.Reader, msgp.Decodable) +// +// There are also methods for converting MessagePack to JSON without +// an explicit de-serialization step. +// +// For additional tips, tricks, and gotchas, please visit +// the wiki at http://github.com/tinylib/msgp +package msgp + +const last4 = 0x0f +const first4 = 0xf0 +const last5 = 0x1f +const first3 = 0xe0 +const last7 = 0x7f + +func isfixint(b byte) bool { + return b>>7 == 0 +} + +func isnfixint(b byte) bool { + return b&first3 == mnfixint +} + +func isfixmap(b byte) bool { + return b&first4 == mfixmap +} + +func isfixarray(b byte) bool { + return b&first4 == mfixarray +} + +func isfixstr(b byte) bool { + return b&first3 == mfixstr +} + +func wfixint(u uint8) byte { + return u & last7 +} + +func rfixint(b byte) uint8 { + return b +} + +func wnfixint(i int8) byte { + return byte(i) | mnfixint +} + +func rnfixint(b byte) int8 { + return int8(b) +} + +func rfixmap(b byte) uint8 { + return b & last4 +} + +func wfixmap(u uint8) byte { + return mfixmap | (u & last4) +} + +func rfixstr(b byte) uint8 { + return b & last5 +} + +func wfixstr(u uint8) byte { + return (u & last5) | mfixstr +} + +func rfixarray(b byte) uint8 { + return (b & last4) +} + +func wfixarray(u uint8) byte { + return (u & last4) | mfixarray +} + +// These are all the byte +// prefixes defined by the +// msgpack standard +const ( + // 0XXXXXXX + mfixint uint8 = 0x00 + + // 111XXXXX + mnfixint uint8 = 0xe0 + + // 1000XXXX + mfixmap uint8 = 0x80 + + // 1001XXXX + mfixarray uint8 = 0x90 + + // 101XXXXX + mfixstr uint8 = 0xa0 + + mnil uint8 = 0xc0 + mfalse uint8 = 0xc2 + mtrue uint8 = 0xc3 + mbin8 uint8 = 0xc4 + mbin16 uint8 = 0xc5 + mbin32 uint8 = 0xc6 + mext8 uint8 = 0xc7 + mext16 uint8 = 0xc8 + mext32 uint8 = 0xc9 + mfloat32 uint8 = 0xca + mfloat64 uint8 = 0xcb + muint8 uint8 = 0xcc + muint16 uint8 = 0xcd + muint32 uint8 = 0xce + muint64 uint8 = 0xcf + mint8 uint8 = 0xd0 + mint16 uint8 = 0xd1 + mint32 uint8 = 0xd2 + mint64 uint8 = 0xd3 + mfixext1 uint8 = 0xd4 + mfixext2 uint8 = 0xd5 + mfixext4 uint8 = 0xd6 + mfixext8 uint8 = 0xd7 + mfixext16 uint8 = 0xd8 + mstr8 uint8 = 0xd9 + mstr16 uint8 = 0xda + mstr32 uint8 = 0xdb + marray16 uint8 = 0xdc + marray32 uint8 = 0xdd + mmap16 uint8 = 0xde + mmap32 uint8 = 0xdf +) diff --git a/vendor/github.com/tinylib/msgp/msgp/edit.go b/vendor/github.com/tinylib/msgp/msgp/edit.go new file mode 100644 index 00000000..b473a6f6 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/edit.go @@ -0,0 +1,242 @@ +package msgp + +import ( + "math" +) + +// Locate returns a []byte pointing to the field +// in a messagepack map with the provided key. (The returned []byte +// points to a sub-slice of 'raw'; Locate does no allocations.) If the +// key doesn't exist in the map, a zero-length []byte will be returned. +func Locate(key string, raw []byte) []byte { + s, n := locate(raw, key) + return raw[s:n] +} + +// Replace takes a key ("key") in a messagepack map ("raw") +// and replaces its value with the one provided and returns +// the new []byte. The returned []byte may point to the same +// memory as "raw". Replace makes no effort to evaluate the validity +// of the contents of 'val'. It may use up to the full capacity of 'raw.' +// Replace returns 'nil' if the field doesn't exist or if the object in 'raw' +// is not a map. +func Replace(key string, raw []byte, val []byte) []byte { + start, end := locate(raw, key) + if start == end { + return nil + } + return replace(raw, start, end, val, true) +} + +// CopyReplace works similarly to Replace except that the returned +// byte slice does not point to the same memory as 'raw'. CopyReplace +// returns 'nil' if the field doesn't exist or 'raw' isn't a map. +func CopyReplace(key string, raw []byte, val []byte) []byte { + start, end := locate(raw, key) + if start == end { + return nil + } + return replace(raw, start, end, val, false) +} + +// Remove removes a key-value pair from 'raw'. It returns +// 'raw' unchanged if the key didn't exist. +func Remove(key string, raw []byte) []byte { + start, end := locateKV(raw, key) + if start == end { + return raw + } + raw = raw[:start+copy(raw[start:], raw[end:])] + return resizeMap(raw, -1) +} + +// HasKey returns whether the map in 'raw' has +// a field with key 'key' +func HasKey(key string, raw []byte) bool { + sz, bts, err := ReadMapHeaderBytes(raw) + if err != nil { + return false + } + var field []byte + for i := uint32(0); i < sz; i++ { + field, bts, err = ReadStringZC(bts) + if err != nil { + return false + } + if UnsafeString(field) == key { + return true + } + } + return false +} + +func replace(raw []byte, start int, end int, val []byte, inplace bool) []byte { + ll := end - start // length of segment to replace + lv := len(val) + + if inplace { + extra := lv - ll + + // fastest case: we're doing + // a 1:1 replacement + if extra == 0 { + copy(raw[start:], val) + return raw + + } else if extra < 0 { + // 'val' smaller than replaced value + // copy in place and shift back + + x := copy(raw[start:], val) + y := copy(raw[start+x:], raw[end:]) + return raw[:start+x+y] + + } else if extra < cap(raw)-len(raw) { + // 'val' less than (cap-len) extra bytes + // copy in place and shift forward + raw = raw[0 : len(raw)+extra] + // shift end forward + copy(raw[end+extra:], raw[end:]) + copy(raw[start:], val) + return raw + } + } + + // we have to allocate new space + out := make([]byte, len(raw)+len(val)-ll) + x := copy(out, raw[:start]) + y := copy(out[x:], val) + copy(out[x+y:], raw[end:]) + return out +} + +// locate does a naive O(n) search for the map key; returns start, end +// (returns 0,0 on error) +func locate(raw []byte, key string) (start int, end int) { + var ( + sz uint32 + bts []byte + field []byte + err error + ) + sz, bts, err = ReadMapHeaderBytes(raw) + if err != nil { + return + } + + // loop and locate field + for i := uint32(0); i < sz; i++ { + field, bts, err = ReadStringZC(bts) + if err != nil { + return 0, 0 + } + if UnsafeString(field) == key { + // start location + l := len(raw) + start = l - len(bts) + bts, err = Skip(bts) + if err != nil { + return 0, 0 + } + end = l - len(bts) + return + } + bts, err = Skip(bts) + if err != nil { + return 0, 0 + } + } + return 0, 0 +} + +// locate key AND value +func locateKV(raw []byte, key string) (start int, end int) { + var ( + sz uint32 + bts []byte + field []byte + err error + ) + sz, bts, err = ReadMapHeaderBytes(raw) + if err != nil { + return 0, 0 + } + + for i := uint32(0); i < sz; i++ { + tmp := len(bts) + field, bts, err = ReadStringZC(bts) + if err != nil { + return 0, 0 + } + if UnsafeString(field) == key { + start = len(raw) - tmp + bts, err = Skip(bts) + if err != nil { + return 0, 0 + } + end = len(raw) - len(bts) + return + } + bts, err = Skip(bts) + if err != nil { + return 0, 0 + } + } + return 0, 0 +} + +// delta is delta on map size +func resizeMap(raw []byte, delta int64) []byte { + var sz int64 + switch raw[0] { + case mmap16: + sz = int64(big.Uint16(raw[1:])) + if sz+delta <= math.MaxUint16 { + big.PutUint16(raw[1:], uint16(sz+delta)) + return raw + } + if cap(raw)-len(raw) >= 2 { + raw = raw[0 : len(raw)+2] + copy(raw[5:], raw[3:]) + raw[0] = mmap32 + big.PutUint32(raw[1:], uint32(sz+delta)) + return raw + } + n := make([]byte, 0, len(raw)+5) + n = AppendMapHeader(n, uint32(sz+delta)) + return append(n, raw[3:]...) + + case mmap32: + sz = int64(big.Uint32(raw[1:])) + big.PutUint32(raw[1:], uint32(sz+delta)) + return raw + + default: + sz = int64(rfixmap(raw[0])) + if sz+delta < 16 { + raw[0] = wfixmap(uint8(sz + delta)) + return raw + } else if sz+delta <= math.MaxUint16 { + if cap(raw)-len(raw) >= 2 { + raw = raw[0 : len(raw)+2] + copy(raw[3:], raw[1:]) + raw[0] = mmap16 + big.PutUint16(raw[1:], uint16(sz+delta)) + return raw + } + n := make([]byte, 0, len(raw)+5) + n = AppendMapHeader(n, uint32(sz+delta)) + return append(n, raw[1:]...) + } + if cap(raw)-len(raw) >= 4 { + raw = raw[0 : len(raw)+4] + copy(raw[5:], raw[1:]) + raw[0] = mmap32 + big.PutUint32(raw[1:], uint32(sz+delta)) + return raw + } + n := make([]byte, 0, len(raw)+5) + n = AppendMapHeader(n, uint32(sz+delta)) + return append(n, raw[1:]...) + } +} diff --git a/vendor/github.com/tinylib/msgp/msgp/elsize.go b/vendor/github.com/tinylib/msgp/msgp/elsize.go new file mode 100644 index 00000000..95762e7e --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/elsize.go @@ -0,0 +1,99 @@ +package msgp + +// size of every object on the wire, +// plus type information. gives us +// constant-time type information +// for traversing composite objects. +// +var sizes = [256]bytespec{ + mnil: {size: 1, extra: constsize, typ: NilType}, + mfalse: {size: 1, extra: constsize, typ: BoolType}, + mtrue: {size: 1, extra: constsize, typ: BoolType}, + mbin8: {size: 2, extra: extra8, typ: BinType}, + mbin16: {size: 3, extra: extra16, typ: BinType}, + mbin32: {size: 5, extra: extra32, typ: BinType}, + mext8: {size: 3, extra: extra8, typ: ExtensionType}, + mext16: {size: 4, extra: extra16, typ: ExtensionType}, + mext32: {size: 6, extra: extra32, typ: ExtensionType}, + mfloat32: {size: 5, extra: constsize, typ: Float32Type}, + mfloat64: {size: 9, extra: constsize, typ: Float64Type}, + muint8: {size: 2, extra: constsize, typ: UintType}, + muint16: {size: 3, extra: constsize, typ: UintType}, + muint32: {size: 5, extra: constsize, typ: UintType}, + muint64: {size: 9, extra: constsize, typ: UintType}, + mint8: {size: 2, extra: constsize, typ: IntType}, + mint16: {size: 3, extra: constsize, typ: IntType}, + mint32: {size: 5, extra: constsize, typ: IntType}, + mint64: {size: 9, extra: constsize, typ: IntType}, + mfixext1: {size: 3, extra: constsize, typ: ExtensionType}, + mfixext2: {size: 4, extra: constsize, typ: ExtensionType}, + mfixext4: {size: 6, extra: constsize, typ: ExtensionType}, + mfixext8: {size: 10, extra: constsize, typ: ExtensionType}, + mfixext16: {size: 18, extra: constsize, typ: ExtensionType}, + mstr8: {size: 2, extra: extra8, typ: StrType}, + mstr16: {size: 3, extra: extra16, typ: StrType}, + mstr32: {size: 5, extra: extra32, typ: StrType}, + marray16: {size: 3, extra: array16v, typ: ArrayType}, + marray32: {size: 5, extra: array32v, typ: ArrayType}, + mmap16: {size: 3, extra: map16v, typ: MapType}, + mmap32: {size: 5, extra: map32v, typ: MapType}, +} + +func init() { + // set up fixed fields + + // fixint + for i := mfixint; i < 0x80; i++ { + sizes[i] = bytespec{size: 1, extra: constsize, typ: IntType} + } + + // nfixint + for i := uint16(mnfixint); i < 0x100; i++ { + sizes[uint8(i)] = bytespec{size: 1, extra: constsize, typ: IntType} + } + + // fixstr gets constsize, + // since the prefix yields the size + for i := mfixstr; i < 0xc0; i++ { + sizes[i] = bytespec{size: 1 + rfixstr(i), extra: constsize, typ: StrType} + } + + // fixmap + for i := mfixmap; i < 0x90; i++ { + sizes[i] = bytespec{size: 1, extra: varmode(2 * rfixmap(i)), typ: MapType} + } + + // fixarray + for i := mfixarray; i < 0xa0; i++ { + sizes[i] = bytespec{size: 1, extra: varmode(rfixarray(i)), typ: ArrayType} + } +} + +// a valid bytespsec has +// non-zero 'size' and +// non-zero 'typ' +type bytespec struct { + size uint8 // prefix size information + extra varmode // extra size information + typ Type // type + _ byte // makes bytespec 4 bytes (yes, this matters) +} + +// size mode +// if positive, # elements for composites +type varmode int8 + +const ( + constsize varmode = 0 // constant size (size bytes + uint8(varmode) objects) + extra8 = -1 // has uint8(p[1]) extra bytes + extra16 = -2 // has be16(p[1:]) extra bytes + extra32 = -3 // has be32(p[1:]) extra bytes + map16v = -4 // use map16 + map32v = -5 // use map32 + array16v = -6 // use array16 + array32v = -7 // use array32 +) + +func getType(v byte) Type { + return sizes[v].typ +} diff --git a/vendor/github.com/tinylib/msgp/msgp/errors.go b/vendor/github.com/tinylib/msgp/msgp/errors.go new file mode 100644 index 00000000..8f197267 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/errors.go @@ -0,0 +1,157 @@ +package msgp + +import ( + "fmt" + "reflect" +) + +var ( + // ErrShortBytes is returned when the + // slice being decoded is too short to + // contain the contents of the message + ErrShortBytes error = errShort{} + + // this error is only returned + // if we reach code that should + // be unreachable + fatal error = errFatal{} +) + +// Error is the interface satisfied +// by all of the errors that originate +// from this package. +type Error interface { + error + + // Resumable returns whether + // or not the error means that + // the stream of data is malformed + // and the information is unrecoverable. + Resumable() bool +} + +type errShort struct{} + +func (e errShort) Error() string { return "msgp: too few bytes left to read object" } +func (e errShort) Resumable() bool { return false } + +type errFatal struct{} + +func (f errFatal) Error() string { return "msgp: fatal decoding error (unreachable code)" } +func (f errFatal) Resumable() bool { return false } + +// ArrayError is an error returned +// when decoding a fix-sized array +// of the wrong size +type ArrayError struct { + Wanted uint32 + Got uint32 +} + +// Error implements the error interface +func (a ArrayError) Error() string { + return fmt.Sprintf("msgp: wanted array of size %d; got %d", a.Wanted, a.Got) +} + +// Resumable is always 'true' for ArrayErrors +func (a ArrayError) Resumable() bool { return true } + +// IntOverflow is returned when a call +// would downcast an integer to a type +// with too few bits to hold its value. +type IntOverflow struct { + Value int64 // the value of the integer + FailedBitsize int // the bit size that the int64 could not fit into +} + +// Error implements the error interface +func (i IntOverflow) Error() string { + return fmt.Sprintf("msgp: %d overflows int%d", i.Value, i.FailedBitsize) +} + +// Resumable is always 'true' for overflows +func (i IntOverflow) Resumable() bool { return true } + +// UintOverflow is returned when a call +// would downcast an unsigned integer to a type +// with too few bits to hold its value +type UintOverflow struct { + Value uint64 // value of the uint + FailedBitsize int // the bit size that couldn't fit the value +} + +// Error implements the error interface +func (u UintOverflow) Error() string { + return fmt.Sprintf("msgp: %d overflows uint%d", u.Value, u.FailedBitsize) +} + +// Resumable is always 'true' for overflows +func (u UintOverflow) Resumable() bool { return true } + +// UintBelowZero is returned when a call +// would cast a signed integer below zero +// to an unsigned integer. +type UintBelowZero struct { + Value int64 // value of the incoming int +} + +// Error implements the error interface +func (u UintBelowZero) Error() string { + return fmt.Sprintf("msgp: attempted to cast int %d to unsigned", u.Value) +} + +// Resumable is always 'true' for overflows +func (u UintBelowZero) Resumable() bool { return true } + +// A TypeError is returned when a particular +// decoding method is unsuitable for decoding +// a particular MessagePack value. +type TypeError struct { + Method Type // Type expected by method + Encoded Type // Type actually encoded +} + +// Error implements the error interface +func (t TypeError) Error() string { + return fmt.Sprintf("msgp: attempted to decode type %q with method for %q", t.Encoded, t.Method) +} + +// Resumable returns 'true' for TypeErrors +func (t TypeError) Resumable() bool { return true } + +// returns either InvalidPrefixError or +// TypeError depending on whether or not +// the prefix is recognized +func badPrefix(want Type, lead byte) error { + t := sizes[lead].typ + if t == InvalidType { + return InvalidPrefixError(lead) + } + return TypeError{Method: want, Encoded: t} +} + +// InvalidPrefixError is returned when a bad encoding +// uses a prefix that is not recognized in the MessagePack standard. +// This kind of error is unrecoverable. +type InvalidPrefixError byte + +// Error implements the error interface +func (i InvalidPrefixError) Error() string { + return fmt.Sprintf("msgp: unrecognized type prefix 0x%x", byte(i)) +} + +// Resumable returns 'false' for InvalidPrefixErrors +func (i InvalidPrefixError) Resumable() bool { return false } + +// ErrUnsupportedType is returned +// when a bad argument is supplied +// to a function that takes `interface{}`. +type ErrUnsupportedType struct { + T reflect.Type +} + +// Error implements error +func (e *ErrUnsupportedType) Error() string { return fmt.Sprintf("msgp: type %q not supported", e.T) } + +// Resumable returns 'true' for ErrUnsupportedType +func (e *ErrUnsupportedType) Resumable() bool { return true } diff --git a/vendor/github.com/tinylib/msgp/msgp/extension.go b/vendor/github.com/tinylib/msgp/msgp/extension.go new file mode 100644 index 00000000..0b31dcdb --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/extension.go @@ -0,0 +1,549 @@ +package msgp + +import ( + "fmt" + "math" +) + +const ( + // Complex64Extension is the extension number used for complex64 + Complex64Extension = 3 + + // Complex128Extension is the extension number used for complex128 + Complex128Extension = 4 + + // TimeExtension is the extension number used for time.Time + TimeExtension = 5 +) + +// our extensions live here +var extensionReg = make(map[int8]func() Extension) + +// RegisterExtension registers extensions so that they +// can be initialized and returned by methods that +// decode `interface{}` values. This should only +// be called during initialization. f() should return +// a newly-initialized zero value of the extension. Keep in +// mind that extensions 3, 4, and 5 are reserved for +// complex64, complex128, and time.Time, respectively, +// and that MessagePack reserves extension types from -127 to -1. +// +// For example, if you wanted to register a user-defined struct: +// +// msgp.RegisterExtension(10, func() msgp.Extension { &MyExtension{} }) +// +// RegisterExtension will panic if you call it multiple times +// with the same 'typ' argument, or if you use a reserved +// type (3, 4, or 5). +func RegisterExtension(typ int8, f func() Extension) { + switch typ { + case Complex64Extension, Complex128Extension, TimeExtension: + panic(fmt.Sprint("msgp: forbidden extension type:", typ)) + } + if _, ok := extensionReg[typ]; ok { + panic(fmt.Sprint("msgp: RegisterExtension() called with typ", typ, "more than once")) + } + extensionReg[typ] = f +} + +// ExtensionTypeError is an error type returned +// when there is a mis-match between an extension type +// and the type encoded on the wire +type ExtensionTypeError struct { + Got int8 + Want int8 +} + +// Error implements the error interface +func (e ExtensionTypeError) Error() string { + return fmt.Sprintf("msgp: error decoding extension: wanted type %d; got type %d", e.Want, e.Got) +} + +// Resumable returns 'true' for ExtensionTypeErrors +func (e ExtensionTypeError) Resumable() bool { return true } + +func errExt(got int8, wanted int8) error { + return ExtensionTypeError{Got: got, Want: wanted} +} + +// Extension is the interface fulfilled +// by types that want to define their +// own binary encoding. +type Extension interface { + // ExtensionType should return + // a int8 that identifies the concrete + // type of the extension. (Types <0 are + // officially reserved by the MessagePack + // specifications.) + ExtensionType() int8 + + // Len should return the length + // of the data to be encoded + Len() int + + // MarshalBinaryTo should copy + // the data into the supplied slice, + // assuming that the slice has length Len() + MarshalBinaryTo([]byte) error + + UnmarshalBinary([]byte) error +} + +// RawExtension implements the Extension interface +type RawExtension struct { + Data []byte + Type int8 +} + +// ExtensionType implements Extension.ExtensionType, and returns r.Type +func (r *RawExtension) ExtensionType() int8 { return r.Type } + +// Len implements Extension.Len, and returns len(r.Data) +func (r *RawExtension) Len() int { return len(r.Data) } + +// MarshalBinaryTo implements Extension.MarshalBinaryTo, +// and returns a copy of r.Data +func (r *RawExtension) MarshalBinaryTo(d []byte) error { + copy(d, r.Data) + return nil +} + +// UnmarshalBinary implements Extension.UnmarshalBinary, +// and sets r.Data to the contents of the provided slice +func (r *RawExtension) UnmarshalBinary(b []byte) error { + if cap(r.Data) >= len(b) { + r.Data = r.Data[0:len(b)] + } else { + r.Data = make([]byte, len(b)) + } + copy(r.Data, b) + return nil +} + +// WriteExtension writes an extension type to the writer +func (mw *Writer) WriteExtension(e Extension) error { + l := e.Len() + var err error + switch l { + case 0: + o, err := mw.require(3) + if err != nil { + return err + } + mw.buf[o] = mext8 + mw.buf[o+1] = 0 + mw.buf[o+2] = byte(e.ExtensionType()) + case 1: + o, err := mw.require(2) + if err != nil { + return err + } + mw.buf[o] = mfixext1 + mw.buf[o+1] = byte(e.ExtensionType()) + case 2: + o, err := mw.require(2) + if err != nil { + return err + } + mw.buf[o] = mfixext2 + mw.buf[o+1] = byte(e.ExtensionType()) + case 4: + o, err := mw.require(2) + if err != nil { + return err + } + mw.buf[o] = mfixext4 + mw.buf[o+1] = byte(e.ExtensionType()) + case 8: + o, err := mw.require(2) + if err != nil { + return err + } + mw.buf[o] = mfixext8 + mw.buf[o+1] = byte(e.ExtensionType()) + case 16: + o, err := mw.require(2) + if err != nil { + return err + } + mw.buf[o] = mfixext16 + mw.buf[o+1] = byte(e.ExtensionType()) + default: + switch { + case l < math.MaxUint8: + o, err := mw.require(3) + if err != nil { + return err + } + mw.buf[o] = mext8 + mw.buf[o+1] = byte(uint8(l)) + mw.buf[o+2] = byte(e.ExtensionType()) + case l < math.MaxUint16: + o, err := mw.require(4) + if err != nil { + return err + } + mw.buf[o] = mext16 + big.PutUint16(mw.buf[o+1:], uint16(l)) + mw.buf[o+3] = byte(e.ExtensionType()) + default: + o, err := mw.require(6) + if err != nil { + return err + } + mw.buf[o] = mext32 + big.PutUint32(mw.buf[o+1:], uint32(l)) + mw.buf[o+5] = byte(e.ExtensionType()) + } + } + // we can only write directly to the + // buffer if we're sure that it + // fits the object + if l <= mw.bufsize() { + o, err := mw.require(l) + if err != nil { + return err + } + return e.MarshalBinaryTo(mw.buf[o:]) + } + // here we create a new buffer + // just large enough for the body + // and save it as the write buffer + err = mw.flush() + if err != nil { + return err + } + buf := make([]byte, l) + err = e.MarshalBinaryTo(buf) + if err != nil { + return err + } + mw.buf = buf + mw.wloc = l + return nil +} + +// peek at the extension type, assuming the next +// kind to be read is Extension +func (m *Reader) peekExtensionType() (int8, error) { + p, err := m.R.Peek(2) + if err != nil { + return 0, err + } + spec := sizes[p[0]] + if spec.typ != ExtensionType { + return 0, badPrefix(ExtensionType, p[0]) + } + if spec.extra == constsize { + return int8(p[1]), nil + } + size := spec.size + p, err = m.R.Peek(int(size)) + if err != nil { + return 0, err + } + return int8(p[size-1]), nil +} + +// peekExtension peeks at the extension encoding type +// (must guarantee at least 1 byte in 'b') +func peekExtension(b []byte) (int8, error) { + spec := sizes[b[0]] + size := spec.size + if spec.typ != ExtensionType { + return 0, badPrefix(ExtensionType, b[0]) + } + if len(b) < int(size) { + return 0, ErrShortBytes + } + // for fixed extensions, + // the type information is in + // the second byte + if spec.extra == constsize { + return int8(b[1]), nil + } + // otherwise, it's in the last + // part of the prefix + return int8(b[size-1]), nil +} + +// ReadExtension reads the next object from the reader +// as an extension. ReadExtension will fail if the next +// object in the stream is not an extension, or if +// e.Type() is not the same as the wire type. +func (m *Reader) ReadExtension(e Extension) (err error) { + var p []byte + p, err = m.R.Peek(2) + if err != nil { + return + } + lead := p[0] + var read int + var off int + switch lead { + case mfixext1: + if int8(p[1]) != e.ExtensionType() { + err = errExt(int8(p[1]), e.ExtensionType()) + return + } + p, err = m.R.Peek(3) + if err != nil { + return + } + err = e.UnmarshalBinary(p[2:]) + if err == nil { + _, err = m.R.Skip(3) + } + return + + case mfixext2: + if int8(p[1]) != e.ExtensionType() { + err = errExt(int8(p[1]), e.ExtensionType()) + return + } + p, err = m.R.Peek(4) + if err != nil { + return + } + err = e.UnmarshalBinary(p[2:]) + if err == nil { + _, err = m.R.Skip(4) + } + return + + case mfixext4: + if int8(p[1]) != e.ExtensionType() { + err = errExt(int8(p[1]), e.ExtensionType()) + return + } + p, err = m.R.Peek(6) + if err != nil { + return + } + err = e.UnmarshalBinary(p[2:]) + if err == nil { + _, err = m.R.Skip(6) + } + return + + case mfixext8: + if int8(p[1]) != e.ExtensionType() { + err = errExt(int8(p[1]), e.ExtensionType()) + return + } + p, err = m.R.Peek(10) + if err != nil { + return + } + err = e.UnmarshalBinary(p[2:]) + if err == nil { + _, err = m.R.Skip(10) + } + return + + case mfixext16: + if int8(p[1]) != e.ExtensionType() { + err = errExt(int8(p[1]), e.ExtensionType()) + return + } + p, err = m.R.Peek(18) + if err != nil { + return + } + err = e.UnmarshalBinary(p[2:]) + if err == nil { + _, err = m.R.Skip(18) + } + return + + case mext8: + p, err = m.R.Peek(3) + if err != nil { + return + } + if int8(p[2]) != e.ExtensionType() { + err = errExt(int8(p[2]), e.ExtensionType()) + return + } + read = int(uint8(p[1])) + off = 3 + + case mext16: + p, err = m.R.Peek(4) + if err != nil { + return + } + if int8(p[3]) != e.ExtensionType() { + err = errExt(int8(p[3]), e.ExtensionType()) + return + } + read = int(big.Uint16(p[1:])) + off = 4 + + case mext32: + p, err = m.R.Peek(6) + if err != nil { + return + } + if int8(p[5]) != e.ExtensionType() { + err = errExt(int8(p[5]), e.ExtensionType()) + return + } + read = int(big.Uint32(p[1:])) + off = 6 + + default: + err = badPrefix(ExtensionType, lead) + return + } + + p, err = m.R.Peek(read + off) + if err != nil { + return + } + err = e.UnmarshalBinary(p[off:]) + if err == nil { + _, err = m.R.Skip(read + off) + } + return +} + +// AppendExtension appends a MessagePack extension to the provided slice +func AppendExtension(b []byte, e Extension) ([]byte, error) { + l := e.Len() + var o []byte + var n int + switch l { + case 0: + o, n = ensure(b, 3) + o[n] = mext8 + o[n+1] = 0 + o[n+2] = byte(e.ExtensionType()) + return o[:n+3], nil + case 1: + o, n = ensure(b, 3) + o[n] = mfixext1 + o[n+1] = byte(e.ExtensionType()) + n += 2 + case 2: + o, n = ensure(b, 4) + o[n] = mfixext2 + o[n+1] = byte(e.ExtensionType()) + n += 2 + case 4: + o, n = ensure(b, 6) + o[n] = mfixext4 + o[n+1] = byte(e.ExtensionType()) + n += 2 + case 8: + o, n = ensure(b, 10) + o[n] = mfixext8 + o[n+1] = byte(e.ExtensionType()) + n += 2 + case 16: + o, n = ensure(b, 18) + o[n] = mfixext16 + o[n+1] = byte(e.ExtensionType()) + n += 2 + default: + switch { + case l < math.MaxUint8: + o, n = ensure(b, l+3) + o[n] = mext8 + o[n+1] = byte(uint8(l)) + o[n+2] = byte(e.ExtensionType()) + n += 3 + case l < math.MaxUint16: + o, n = ensure(b, l+4) + o[n] = mext16 + big.PutUint16(o[n+1:], uint16(l)) + o[n+3] = byte(e.ExtensionType()) + n += 4 + default: + o, n = ensure(b, l+6) + o[n] = mext32 + big.PutUint32(o[n+1:], uint32(l)) + o[n+5] = byte(e.ExtensionType()) + n += 6 + } + } + return o, e.MarshalBinaryTo(o[n:]) +} + +// ReadExtensionBytes reads an extension from 'b' into 'e' +// and returns any remaining bytes. +// Possible errors: +// - ErrShortBytes ('b' not long enough) +// - ExtensionTypeErorr{} (wire type not the same as e.Type()) +// - TypeErorr{} (next object not an extension) +// - InvalidPrefixError +// - An umarshal error returned from e.UnmarshalBinary +func ReadExtensionBytes(b []byte, e Extension) ([]byte, error) { + l := len(b) + if l < 3 { + return b, ErrShortBytes + } + lead := b[0] + var ( + sz int // size of 'data' + off int // offset of 'data' + typ int8 + ) + switch lead { + case mfixext1: + typ = int8(b[1]) + sz = 1 + off = 2 + case mfixext2: + typ = int8(b[1]) + sz = 2 + off = 2 + case mfixext4: + typ = int8(b[1]) + sz = 4 + off = 2 + case mfixext8: + typ = int8(b[1]) + sz = 8 + off = 2 + case mfixext16: + typ = int8(b[1]) + sz = 16 + off = 2 + case mext8: + sz = int(uint8(b[1])) + typ = int8(b[2]) + off = 3 + if sz == 0 { + return b[3:], e.UnmarshalBinary(b[3:3]) + } + case mext16: + if l < 4 { + return b, ErrShortBytes + } + sz = int(big.Uint16(b[1:])) + typ = int8(b[3]) + off = 4 + case mext32: + if l < 6 { + return b, ErrShortBytes + } + sz = int(big.Uint32(b[1:])) + typ = int8(b[5]) + off = 6 + default: + return b, badPrefix(ExtensionType, lead) + } + + if typ != e.ExtensionType() { + return b, errExt(typ, e.ExtensionType()) + } + + // the data of the extension starts + // at 'off' and is 'sz' bytes long + if len(b[off:]) < sz { + return b, ErrShortBytes + } + tot := off + sz + return b[tot:], e.UnmarshalBinary(b[off:tot]) +} diff --git a/vendor/github.com/tinylib/msgp/msgp/file.go b/vendor/github.com/tinylib/msgp/msgp/file.go new file mode 100644 index 00000000..8e7370eb --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/file.go @@ -0,0 +1,92 @@ +// +build linux darwin dragonfly freebsd netbsd openbsd +// +build !appengine + +package msgp + +import ( + "os" + "syscall" +) + +// ReadFile reads a file into 'dst' using +// a read-only memory mapping. Consequently, +// the file must be mmap-able, and the +// Unmarshaler should never write to +// the source memory. (Methods generated +// by the msgp tool obey that constraint, but +// user-defined implementations may not.) +// +// Reading and writing through file mappings +// is only efficient for large files; small +// files are best read and written using +// the ordinary streaming interfaces. +// +func ReadFile(dst Unmarshaler, file *os.File) error { + stat, err := file.Stat() + if err != nil { + return err + } + data, err := syscall.Mmap(int(file.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return err + } + adviseRead(data) + _, err = dst.UnmarshalMsg(data) + uerr := syscall.Munmap(data) + if err == nil { + err = uerr + } + return err +} + +// MarshalSizer is the combination +// of the Marshaler and Sizer +// interfaces. +type MarshalSizer interface { + Marshaler + Sizer +} + +// WriteFile writes a file from 'src' using +// memory mapping. It overwrites the entire +// contents of the previous file. +// The mapping size is calculated +// using the `Msgsize()` method +// of 'src', so it must produce a result +// equal to or greater than the actual encoded +// size of the object. Otherwise, +// a fault (SIGBUS) will occur. +// +// Reading and writing through file mappings +// is only efficient for large files; small +// files are best read and written using +// the ordinary streaming interfaces. +// +// NOTE: The performance of this call +// is highly OS- and filesystem-dependent. +// Users should take care to test that this +// performs as expected in a production environment. +// (Linux users should run a kernel and filesystem +// that support fallocate(2) for the best results.) +func WriteFile(src MarshalSizer, file *os.File) error { + sz := src.Msgsize() + err := fallocate(file, int64(sz)) + if err != nil { + return err + } + data, err := syscall.Mmap(int(file.Fd()), 0, sz, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) + if err != nil { + return err + } + adviseWrite(data) + chunk := data[:0] + chunk, err = src.MarshalMsg(chunk) + if err != nil { + return err + } + uerr := syscall.Munmap(data) + if uerr != nil { + return uerr + } + return file.Truncate(int64(len(chunk))) +} diff --git a/vendor/github.com/tinylib/msgp/msgp/file_port.go b/vendor/github.com/tinylib/msgp/msgp/file_port.go new file mode 100644 index 00000000..6e654dbd --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/file_port.go @@ -0,0 +1,47 @@ +// +build windows appengine + +package msgp + +import ( + "io/ioutil" + "os" +) + +// MarshalSizer is the combination +// of the Marshaler and Sizer +// interfaces. +type MarshalSizer interface { + Marshaler + Sizer +} + +func ReadFile(dst Unmarshaler, file *os.File) error { + if u, ok := dst.(Decodable); ok { + return u.DecodeMsg(NewReader(file)) + } + + data, err := ioutil.ReadAll(file) + if err != nil { + return err + } + _, err = dst.UnmarshalMsg(data) + return err +} + +func WriteFile(src MarshalSizer, file *os.File) error { + if e, ok := src.(Encodable); ok { + w := NewWriter(file) + err := e.EncodeMsg(w) + if err == nil { + err = w.Flush() + } + return err + } + + raw, err := src.MarshalMsg(nil) + if err != nil { + return err + } + _, err = file.Write(raw) + return err +} diff --git a/vendor/github.com/tinylib/msgp/msgp/integers.go b/vendor/github.com/tinylib/msgp/msgp/integers.go new file mode 100644 index 00000000..f817d775 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/integers.go @@ -0,0 +1,174 @@ +package msgp + +/* ---------------------------------- + integer encoding utilities + (inline-able) + + TODO(tinylib): there are faster, + albeit non-portable solutions + to the code below. implement + byteswap? + ---------------------------------- */ + +func putMint64(b []byte, i int64) { + b[0] = mint64 + b[1] = byte(i >> 56) + b[2] = byte(i >> 48) + b[3] = byte(i >> 40) + b[4] = byte(i >> 32) + b[5] = byte(i >> 24) + b[6] = byte(i >> 16) + b[7] = byte(i >> 8) + b[8] = byte(i) +} + +func getMint64(b []byte) int64 { + return (int64(b[1]) << 56) | (int64(b[2]) << 48) | + (int64(b[3]) << 40) | (int64(b[4]) << 32) | + (int64(b[5]) << 24) | (int64(b[6]) << 16) | + (int64(b[7]) << 8) | (int64(b[8])) +} + +func putMint32(b []byte, i int32) { + b[0] = mint32 + b[1] = byte(i >> 24) + b[2] = byte(i >> 16) + b[3] = byte(i >> 8) + b[4] = byte(i) +} + +func getMint32(b []byte) int32 { + return (int32(b[1]) << 24) | (int32(b[2]) << 16) | (int32(b[3]) << 8) | (int32(b[4])) +} + +func putMint16(b []byte, i int16) { + b[0] = mint16 + b[1] = byte(i >> 8) + b[2] = byte(i) +} + +func getMint16(b []byte) (i int16) { + return (int16(b[1]) << 8) | int16(b[2]) +} + +func putMint8(b []byte, i int8) { + b[0] = mint8 + b[1] = byte(i) +} + +func getMint8(b []byte) (i int8) { + return int8(b[1]) +} + +func putMuint64(b []byte, u uint64) { + b[0] = muint64 + b[1] = byte(u >> 56) + b[2] = byte(u >> 48) + b[3] = byte(u >> 40) + b[4] = byte(u >> 32) + b[5] = byte(u >> 24) + b[6] = byte(u >> 16) + b[7] = byte(u >> 8) + b[8] = byte(u) +} + +func getMuint64(b []byte) uint64 { + return (uint64(b[1]) << 56) | (uint64(b[2]) << 48) | + (uint64(b[3]) << 40) | (uint64(b[4]) << 32) | + (uint64(b[5]) << 24) | (uint64(b[6]) << 16) | + (uint64(b[7]) << 8) | (uint64(b[8])) +} + +func putMuint32(b []byte, u uint32) { + b[0] = muint32 + b[1] = byte(u >> 24) + b[2] = byte(u >> 16) + b[3] = byte(u >> 8) + b[4] = byte(u) +} + +func getMuint32(b []byte) uint32 { + return (uint32(b[1]) << 24) | (uint32(b[2]) << 16) | (uint32(b[3]) << 8) | (uint32(b[4])) +} + +func putMuint16(b []byte, u uint16) { + b[0] = muint16 + b[1] = byte(u >> 8) + b[2] = byte(u) +} + +func getMuint16(b []byte) uint16 { + return (uint16(b[1]) << 8) | uint16(b[2]) +} + +func putMuint8(b []byte, u uint8) { + b[0] = muint8 + b[1] = byte(u) +} + +func getMuint8(b []byte) uint8 { + return uint8(b[1]) +} + +func getUnix(b []byte) (sec int64, nsec int32) { + sec = (int64(b[0]) << 56) | (int64(b[1]) << 48) | + (int64(b[2]) << 40) | (int64(b[3]) << 32) | + (int64(b[4]) << 24) | (int64(b[5]) << 16) | + (int64(b[6]) << 8) | (int64(b[7])) + + nsec = (int32(b[8]) << 24) | (int32(b[9]) << 16) | (int32(b[10]) << 8) | (int32(b[11])) + return +} + +func putUnix(b []byte, sec int64, nsec int32) { + b[0] = byte(sec >> 56) + b[1] = byte(sec >> 48) + b[2] = byte(sec >> 40) + b[3] = byte(sec >> 32) + b[4] = byte(sec >> 24) + b[5] = byte(sec >> 16) + b[6] = byte(sec >> 8) + b[7] = byte(sec) + b[8] = byte(nsec >> 24) + b[9] = byte(nsec >> 16) + b[10] = byte(nsec >> 8) + b[11] = byte(nsec) +} + +/* ----------------------------- + prefix utilities + ----------------------------- */ + +// write prefix and uint8 +func prefixu8(b []byte, pre byte, sz uint8) { + b[0] = pre + b[1] = byte(sz) +} + +// write prefix and big-endian uint16 +func prefixu16(b []byte, pre byte, sz uint16) { + b[0] = pre + b[1] = byte(sz >> 8) + b[2] = byte(sz) +} + +// write prefix and big-endian uint32 +func prefixu32(b []byte, pre byte, sz uint32) { + b[0] = pre + b[1] = byte(sz >> 24) + b[2] = byte(sz >> 16) + b[3] = byte(sz >> 8) + b[4] = byte(sz) +} + +func prefixu64(b []byte, pre byte, sz uint64) { + b[0] = pre + b[1] = byte(sz >> 56) + b[2] = byte(sz >> 48) + b[3] = byte(sz >> 40) + b[4] = byte(sz >> 32) + b[5] = byte(sz >> 24) + b[6] = byte(sz >> 16) + b[7] = byte(sz >> 8) + b[8] = byte(sz) +} diff --git a/vendor/github.com/tinylib/msgp/msgp/json.go b/vendor/github.com/tinylib/msgp/msgp/json.go new file mode 100644 index 00000000..4325860a --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/json.go @@ -0,0 +1,542 @@ +package msgp + +import ( + "bufio" + "encoding/base64" + "encoding/json" + "io" + "strconv" + "unicode/utf8" +) + +var ( + null = []byte("null") + hex = []byte("0123456789abcdef") +) + +var defuns [_maxtype]func(jsWriter, *Reader) (int, error) + +// note: there is an initialization loop if +// this isn't set up during init() +func init() { + // since none of these functions are inline-able, + // there is not much of a penalty to the indirect + // call. however, this is best expressed as a jump-table... + defuns = [_maxtype]func(jsWriter, *Reader) (int, error){ + StrType: rwString, + BinType: rwBytes, + MapType: rwMap, + ArrayType: rwArray, + Float64Type: rwFloat64, + Float32Type: rwFloat32, + BoolType: rwBool, + IntType: rwInt, + UintType: rwUint, + NilType: rwNil, + ExtensionType: rwExtension, + Complex64Type: rwExtension, + Complex128Type: rwExtension, + TimeType: rwTime, + } +} + +// this is the interface +// used to write json +type jsWriter interface { + io.Writer + io.ByteWriter + WriteString(string) (int, error) +} + +// CopyToJSON reads MessagePack from 'src' and copies it +// as JSON to 'dst' until EOF. +func CopyToJSON(dst io.Writer, src io.Reader) (n int64, err error) { + r := NewReader(src) + n, err = r.WriteToJSON(dst) + freeR(r) + return +} + +// WriteToJSON translates MessagePack from 'r' and writes it as +// JSON to 'w' until the underlying reader returns io.EOF. It returns +// the number of bytes written, and an error if it stopped before EOF. +func (r *Reader) WriteToJSON(w io.Writer) (n int64, err error) { + var j jsWriter + var bf *bufio.Writer + if jsw, ok := w.(jsWriter); ok { + j = jsw + } else { + bf = bufio.NewWriter(w) + j = bf + } + var nn int + for err == nil { + nn, err = rwNext(j, r) + n += int64(nn) + } + if err != io.EOF { + if bf != nil { + bf.Flush() + } + return + } + err = nil + if bf != nil { + err = bf.Flush() + } + return +} + +func rwNext(w jsWriter, src *Reader) (int, error) { + t, err := src.NextType() + if err != nil { + return 0, err + } + return defuns[t](w, src) +} + +func rwMap(dst jsWriter, src *Reader) (n int, err error) { + var comma bool + var sz uint32 + var field []byte + + sz, err = src.ReadMapHeader() + if err != nil { + return + } + + if sz == 0 { + return dst.WriteString("{}") + } + + err = dst.WriteByte('{') + if err != nil { + return + } + n++ + var nn int + for i := uint32(0); i < sz; i++ { + if comma { + err = dst.WriteByte(',') + if err != nil { + return + } + n++ + } + + field, err = src.ReadMapKeyPtr() + if err != nil { + return + } + nn, err = rwquoted(dst, field) + n += nn + if err != nil { + return + } + + err = dst.WriteByte(':') + if err != nil { + return + } + n++ + nn, err = rwNext(dst, src) + n += nn + if err != nil { + return + } + if !comma { + comma = true + } + } + + err = dst.WriteByte('}') + if err != nil { + return + } + n++ + return +} + +func rwArray(dst jsWriter, src *Reader) (n int, err error) { + err = dst.WriteByte('[') + if err != nil { + return + } + var sz uint32 + var nn int + sz, err = src.ReadArrayHeader() + if err != nil { + return + } + comma := false + for i := uint32(0); i < sz; i++ { + if comma { + err = dst.WriteByte(',') + if err != nil { + return + } + n++ + } + nn, err = rwNext(dst, src) + n += nn + if err != nil { + return + } + comma = true + } + + err = dst.WriteByte(']') + if err != nil { + return + } + n++ + return +} + +func rwNil(dst jsWriter, src *Reader) (int, error) { + err := src.ReadNil() + if err != nil { + return 0, err + } + return dst.Write(null) +} + +func rwFloat32(dst jsWriter, src *Reader) (int, error) { + f, err := src.ReadFloat32() + if err != nil { + return 0, err + } + src.scratch = strconv.AppendFloat(src.scratch[:0], float64(f), 'f', -1, 64) + return dst.Write(src.scratch) +} + +func rwFloat64(dst jsWriter, src *Reader) (int, error) { + f, err := src.ReadFloat64() + if err != nil { + return 0, err + } + src.scratch = strconv.AppendFloat(src.scratch[:0], f, 'f', -1, 32) + return dst.Write(src.scratch) +} + +func rwInt(dst jsWriter, src *Reader) (int, error) { + i, err := src.ReadInt64() + if err != nil { + return 0, err + } + src.scratch = strconv.AppendInt(src.scratch[:0], i, 10) + return dst.Write(src.scratch) +} + +func rwUint(dst jsWriter, src *Reader) (int, error) { + u, err := src.ReadUint64() + if err != nil { + return 0, err + } + src.scratch = strconv.AppendUint(src.scratch[:0], u, 10) + return dst.Write(src.scratch) +} + +func rwBool(dst jsWriter, src *Reader) (int, error) { + b, err := src.ReadBool() + if err != nil { + return 0, err + } + if b { + return dst.WriteString("true") + } + return dst.WriteString("false") +} + +func rwTime(dst jsWriter, src *Reader) (int, error) { + t, err := src.ReadTime() + if err != nil { + return 0, err + } + bts, err := t.MarshalJSON() + if err != nil { + return 0, err + } + return dst.Write(bts) +} + +func rwExtension(dst jsWriter, src *Reader) (n int, err error) { + et, err := src.peekExtensionType() + if err != nil { + return 0, err + } + + // registered extensions can override + // the JSON encoding + if j, ok := extensionReg[et]; ok { + var bts []byte + e := j() + err = src.ReadExtension(e) + if err != nil { + return + } + bts, err = json.Marshal(e) + if err != nil { + return + } + return dst.Write(bts) + } + + e := RawExtension{} + e.Type = et + err = src.ReadExtension(&e) + if err != nil { + return + } + + var nn int + err = dst.WriteByte('{') + if err != nil { + return + } + n++ + + nn, err = dst.WriteString(`"type:"`) + n += nn + if err != nil { + return + } + + src.scratch = strconv.AppendInt(src.scratch[0:0], int64(e.Type), 10) + nn, err = dst.Write(src.scratch) + n += nn + if err != nil { + return + } + + nn, err = dst.WriteString(`,"data":"`) + n += nn + if err != nil { + return + } + + enc := base64.NewEncoder(base64.StdEncoding, dst) + + nn, err = enc.Write(e.Data) + n += nn + if err != nil { + return + } + err = enc.Close() + if err != nil { + return + } + nn, err = dst.WriteString(`"}`) + n += nn + return +} + +func rwString(dst jsWriter, src *Reader) (n int, err error) { + var p []byte + p, err = src.R.Peek(1) + if err != nil { + return + } + lead := p[0] + var read int + + if isfixstr(lead) { + read = int(rfixstr(lead)) + src.R.Skip(1) + goto write + } + + switch lead { + case mstr8: + p, err = src.R.Next(2) + if err != nil { + return + } + read = int(uint8(p[1])) + case mstr16: + p, err = src.R.Next(3) + if err != nil { + return + } + read = int(big.Uint16(p[1:])) + case mstr32: + p, err = src.R.Next(5) + if err != nil { + return + } + read = int(big.Uint32(p[1:])) + default: + err = badPrefix(StrType, lead) + return + } +write: + p, err = src.R.Next(read) + if err != nil { + return + } + n, err = rwquoted(dst, p) + return +} + +func rwBytes(dst jsWriter, src *Reader) (n int, err error) { + var nn int + err = dst.WriteByte('"') + if err != nil { + return + } + n++ + src.scratch, err = src.ReadBytes(src.scratch[:0]) + if err != nil { + return + } + enc := base64.NewEncoder(base64.StdEncoding, dst) + nn, err = enc.Write(src.scratch) + n += nn + if err != nil { + return + } + err = enc.Close() + if err != nil { + return + } + err = dst.WriteByte('"') + if err != nil { + return + } + n++ + return +} + +// Below (c) The Go Authors, 2009-2014 +// Subject to the BSD-style license found at http://golang.org +// +// see: encoding/json/encode.go:(*encodeState).stringbytes() +func rwquoted(dst jsWriter, s []byte) (n int, err error) { + var nn int + err = dst.WriteByte('"') + if err != nil { + return + } + n++ + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + if start < i { + nn, err = dst.Write(s[start:i]) + n += nn + if err != nil { + return + } + } + switch b { + case '\\', '"': + err = dst.WriteByte('\\') + if err != nil { + return + } + n++ + err = dst.WriteByte(b) + if err != nil { + return + } + n++ + case '\n': + err = dst.WriteByte('\\') + if err != nil { + return + } + n++ + err = dst.WriteByte('n') + if err != nil { + return + } + n++ + case '\r': + err = dst.WriteByte('\\') + if err != nil { + return + } + n++ + err = dst.WriteByte('r') + if err != nil { + return + } + n++ + default: + nn, err = dst.WriteString(`\u00`) + n += nn + if err != nil { + return + } + err = dst.WriteByte(hex[b>>4]) + if err != nil { + return + } + n++ + err = dst.WriteByte(hex[b&0xF]) + if err != nil { + return + } + n++ + } + i++ + start = i + continue + } + c, size := utf8.DecodeRune(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + nn, err = dst.Write(s[start:i]) + n += nn + if err != nil { + return + } + nn, err = dst.WriteString(`\ufffd`) + n += nn + if err != nil { + return + } + i += size + start = i + continue + } + } + if c == '\u2028' || c == '\u2029' { + if start < i { + nn, err = dst.Write(s[start:i]) + n += nn + if err != nil { + return + } + nn, err = dst.WriteString(`\u202`) + n += nn + if err != nil { + return + } + err = dst.WriteByte(hex[c&0xF]) + if err != nil { + return + } + n++ + } + } + i += size + } + if start < len(s) { + nn, err = dst.Write(s[start:]) + n += nn + if err != nil { + return + } + } + err = dst.WriteByte('"') + if err != nil { + return + } + n++ + return +} diff --git a/vendor/github.com/tinylib/msgp/msgp/json_bytes.go b/vendor/github.com/tinylib/msgp/msgp/json_bytes.go new file mode 100644 index 00000000..438caf53 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/json_bytes.go @@ -0,0 +1,363 @@ +package msgp + +import ( + "bufio" + "encoding/base64" + "encoding/json" + "io" + "strconv" + "time" +) + +var unfuns [_maxtype]func(jsWriter, []byte, []byte) ([]byte, []byte, error) + +func init() { + + // NOTE(pmh): this is best expressed as a jump table, + // but gc doesn't do that yet. revisit post-go1.5. + unfuns = [_maxtype]func(jsWriter, []byte, []byte) ([]byte, []byte, error){ + StrType: rwStringBytes, + BinType: rwBytesBytes, + MapType: rwMapBytes, + ArrayType: rwArrayBytes, + Float64Type: rwFloat64Bytes, + Float32Type: rwFloat32Bytes, + BoolType: rwBoolBytes, + IntType: rwIntBytes, + UintType: rwUintBytes, + NilType: rwNullBytes, + ExtensionType: rwExtensionBytes, + Complex64Type: rwExtensionBytes, + Complex128Type: rwExtensionBytes, + TimeType: rwTimeBytes, + } +} + +// UnmarshalAsJSON takes raw messagepack and writes +// it as JSON to 'w'. If an error is returned, the +// bytes not translated will also be returned. If +// no errors are encountered, the length of the returned +// slice will be zero. +func UnmarshalAsJSON(w io.Writer, msg []byte) ([]byte, error) { + var ( + scratch []byte + cast bool + dst jsWriter + err error + ) + if jsw, ok := w.(jsWriter); ok { + dst = jsw + cast = true + } else { + dst = bufio.NewWriterSize(w, 512) + } + for len(msg) > 0 && err == nil { + msg, scratch, err = writeNext(dst, msg, scratch) + } + if !cast && err == nil { + err = dst.(*bufio.Writer).Flush() + } + return msg, err +} + +func writeNext(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + if len(msg) < 1 { + return msg, scratch, ErrShortBytes + } + t := getType(msg[0]) + if t == InvalidType { + return msg, scratch, InvalidPrefixError(msg[0]) + } + if t == ExtensionType { + et, err := peekExtension(msg) + if err != nil { + return nil, scratch, err + } + if et == TimeExtension { + t = TimeType + } + } + return unfuns[t](w, msg, scratch) +} + +func rwArrayBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + sz, msg, err := ReadArrayHeaderBytes(msg) + if err != nil { + return msg, scratch, err + } + err = w.WriteByte('[') + if err != nil { + return msg, scratch, err + } + for i := uint32(0); i < sz; i++ { + if i != 0 { + err = w.WriteByte(',') + if err != nil { + return msg, scratch, err + } + } + msg, scratch, err = writeNext(w, msg, scratch) + if err != nil { + return msg, scratch, err + } + } + err = w.WriteByte(']') + return msg, scratch, err +} + +func rwMapBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + sz, msg, err := ReadMapHeaderBytes(msg) + if err != nil { + return msg, scratch, err + } + err = w.WriteByte('{') + if err != nil { + return msg, scratch, err + } + for i := uint32(0); i < sz; i++ { + if i != 0 { + err = w.WriteByte(',') + if err != nil { + return msg, scratch, err + } + } + msg, scratch, err = rwMapKeyBytes(w, msg, scratch) + if err != nil { + return msg, scratch, err + } + err = w.WriteByte(':') + if err != nil { + return msg, scratch, err + } + msg, scratch, err = writeNext(w, msg, scratch) + if err != nil { + return msg, scratch, err + } + } + err = w.WriteByte('}') + return msg, scratch, err +} + +func rwMapKeyBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + msg, scratch, err := rwStringBytes(w, msg, scratch) + if err != nil { + if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { + return rwBytesBytes(w, msg, scratch) + } + } + return msg, scratch, err +} + +func rwStringBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + str, msg, err := ReadStringZC(msg) + if err != nil { + return msg, scratch, err + } + _, err = rwquoted(w, str) + return msg, scratch, err +} + +func rwBytesBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + bts, msg, err := ReadBytesZC(msg) + if err != nil { + return msg, scratch, err + } + l := base64.StdEncoding.EncodedLen(len(bts)) + if cap(scratch) >= l { + scratch = scratch[0:l] + } else { + scratch = make([]byte, l) + } + base64.StdEncoding.Encode(scratch, bts) + err = w.WriteByte('"') + if err != nil { + return msg, scratch, err + } + _, err = w.Write(scratch) + if err != nil { + return msg, scratch, err + } + err = w.WriteByte('"') + return msg, scratch, err +} + +func rwNullBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + msg, err := ReadNilBytes(msg) + if err != nil { + return msg, scratch, err + } + _, err = w.Write(null) + return msg, scratch, err +} + +func rwBoolBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + b, msg, err := ReadBoolBytes(msg) + if err != nil { + return msg, scratch, err + } + if b { + _, err = w.WriteString("true") + return msg, scratch, err + } + _, err = w.WriteString("false") + return msg, scratch, err +} + +func rwIntBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + i, msg, err := ReadInt64Bytes(msg) + if err != nil { + return msg, scratch, err + } + scratch = strconv.AppendInt(scratch[0:0], i, 10) + _, err = w.Write(scratch) + return msg, scratch, err +} + +func rwUintBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + u, msg, err := ReadUint64Bytes(msg) + if err != nil { + return msg, scratch, err + } + scratch = strconv.AppendUint(scratch[0:0], u, 10) + _, err = w.Write(scratch) + return msg, scratch, err +} + +func rwFloatBytes(w jsWriter, msg []byte, f64 bool, scratch []byte) ([]byte, []byte, error) { + var f float64 + var err error + var sz int + if f64 { + sz = 64 + f, msg, err = ReadFloat64Bytes(msg) + } else { + sz = 32 + var v float32 + v, msg, err = ReadFloat32Bytes(msg) + f = float64(v) + } + if err != nil { + return msg, scratch, err + } + scratch = strconv.AppendFloat(scratch, f, 'f', -1, sz) + _, err = w.Write(scratch) + return msg, scratch, err +} + +func rwFloat32Bytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + var f float32 + var err error + f, msg, err = ReadFloat32Bytes(msg) + if err != nil { + return msg, scratch, err + } + scratch = strconv.AppendFloat(scratch[:0], float64(f), 'f', -1, 32) + _, err = w.Write(scratch) + return msg, scratch, err +} + +func rwFloat64Bytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + var f float64 + var err error + f, msg, err = ReadFloat64Bytes(msg) + if err != nil { + return msg, scratch, err + } + scratch = strconv.AppendFloat(scratch[:0], f, 'f', -1, 64) + _, err = w.Write(scratch) + return msg, scratch, err +} + +func rwTimeBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + var t time.Time + var err error + t, msg, err = ReadTimeBytes(msg) + if err != nil { + return msg, scratch, err + } + bts, err := t.MarshalJSON() + if err != nil { + return msg, scratch, err + } + _, err = w.Write(bts) + return msg, scratch, err +} + +func rwExtensionBytes(w jsWriter, msg []byte, scratch []byte) ([]byte, []byte, error) { + var err error + var et int8 + et, err = peekExtension(msg) + if err != nil { + return msg, scratch, err + } + + // if it's time.Time + if et == TimeExtension { + var tm time.Time + tm, msg, err = ReadTimeBytes(msg) + if err != nil { + return msg, scratch, err + } + bts, err := tm.MarshalJSON() + if err != nil { + return msg, scratch, err + } + _, err = w.Write(bts) + return msg, scratch, err + } + + // if the extension is registered, + // use its canonical JSON form + if f, ok := extensionReg[et]; ok { + e := f() + msg, err = ReadExtensionBytes(msg, e) + if err != nil { + return msg, scratch, err + } + bts, err := json.Marshal(e) + if err != nil { + return msg, scratch, err + } + _, err = w.Write(bts) + return msg, scratch, err + } + + // otherwise, write `{"type": , "data": ""}` + r := RawExtension{} + r.Type = et + msg, err = ReadExtensionBytes(msg, &r) + if err != nil { + return msg, scratch, err + } + scratch, err = writeExt(w, r, scratch) + return msg, scratch, err +} + +func writeExt(w jsWriter, r RawExtension, scratch []byte) ([]byte, error) { + _, err := w.WriteString(`{"type":`) + if err != nil { + return scratch, err + } + scratch = strconv.AppendInt(scratch[0:0], int64(r.Type), 10) + _, err = w.Write(scratch) + if err != nil { + return scratch, err + } + _, err = w.WriteString(`,"data":"`) + if err != nil { + return scratch, err + } + l := base64.StdEncoding.EncodedLen(len(r.Data)) + if cap(scratch) >= l { + scratch = scratch[0:l] + } else { + scratch = make([]byte, l) + } + base64.StdEncoding.Encode(scratch, r.Data) + _, err = w.Write(scratch) + if err != nil { + return scratch, err + } + _, err = w.WriteString(`"}`) + return scratch, err +} diff --git a/vendor/github.com/tinylib/msgp/msgp/number.go b/vendor/github.com/tinylib/msgp/msgp/number.go new file mode 100644 index 00000000..ad07ef99 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/number.go @@ -0,0 +1,267 @@ +package msgp + +import ( + "math" + "strconv" +) + +// The portable parts of the Number implementation + +// Number can be +// an int64, uint64, float32, +// or float64 internally. +// It can decode itself +// from any of the native +// messagepack number types. +// The zero-value of Number +// is Int(0). Using the equality +// operator with Number compares +// both the type and the value +// of the number. +type Number struct { + // internally, this + // is just a tagged union. + // the raw bits of the number + // are stored the same way regardless. + bits uint64 + typ Type +} + +// AsInt sets the number to an int64. +func (n *Number) AsInt(i int64) { + + // we always store int(0) + // as {0, InvalidType} in + // order to preserve + // the behavior of the == operator + if i == 0 { + n.typ = InvalidType + n.bits = 0 + return + } + + n.typ = IntType + n.bits = uint64(i) +} + +// AsUint sets the number to a uint64. +func (n *Number) AsUint(u uint64) { + n.typ = UintType + n.bits = u +} + +// AsFloat32 sets the value of the number +// to a float32. +func (n *Number) AsFloat32(f float32) { + n.typ = Float32Type + n.bits = uint64(math.Float32bits(f)) +} + +// AsFloat64 sets the value of the +// number to a float64. +func (n *Number) AsFloat64(f float64) { + n.typ = Float64Type + n.bits = math.Float64bits(f) +} + +// Int casts the number as an int64, and +// returns whether or not that was the +// underlying type. +func (n *Number) Int() (int64, bool) { + return int64(n.bits), n.typ == IntType || n.typ == InvalidType +} + +// Uint casts the number as a uint64, and returns +// whether or not that was the underlying type. +func (n *Number) Uint() (uint64, bool) { + return n.bits, n.typ == UintType +} + +// Float casts the number to a float64, and +// returns whether or not that was the underlying +// type (either a float64 or a float32). +func (n *Number) Float() (float64, bool) { + switch n.typ { + case Float32Type: + return float64(math.Float32frombits(uint32(n.bits))), true + case Float64Type: + return math.Float64frombits(n.bits), true + default: + return 0.0, false + } +} + +// Type will return one of: +// Float64Type, Float32Type, UintType, or IntType. +func (n *Number) Type() Type { + if n.typ == InvalidType { + return IntType + } + return n.typ +} + +// DecodeMsg implements msgp.Decodable +func (n *Number) DecodeMsg(r *Reader) error { + typ, err := r.NextType() + if err != nil { + return err + } + switch typ { + case Float32Type: + f, err := r.ReadFloat32() + if err != nil { + return err + } + n.AsFloat32(f) + return nil + case Float64Type: + f, err := r.ReadFloat64() + if err != nil { + return err + } + n.AsFloat64(f) + return nil + case IntType: + i, err := r.ReadInt64() + if err != nil { + return err + } + n.AsInt(i) + return nil + case UintType: + u, err := r.ReadUint64() + if err != nil { + return err + } + n.AsUint(u) + return nil + default: + return TypeError{Encoded: typ, Method: IntType} + } +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (n *Number) UnmarshalMsg(b []byte) ([]byte, error) { + typ := NextType(b) + switch typ { + case IntType: + i, o, err := ReadInt64Bytes(b) + if err != nil { + return b, err + } + n.AsInt(i) + return o, nil + case UintType: + u, o, err := ReadUint64Bytes(b) + if err != nil { + return b, err + } + n.AsUint(u) + return o, nil + case Float64Type: + f, o, err := ReadFloat64Bytes(b) + if err != nil { + return b, err + } + n.AsFloat64(f) + return o, nil + case Float32Type: + f, o, err := ReadFloat32Bytes(b) + if err != nil { + return b, err + } + n.AsFloat32(f) + return o, nil + default: + return b, TypeError{Method: IntType, Encoded: typ} + } +} + +// MarshalMsg implements msgp.Marshaler +func (n *Number) MarshalMsg(b []byte) ([]byte, error) { + switch n.typ { + case IntType: + return AppendInt64(b, int64(n.bits)), nil + case UintType: + return AppendUint64(b, uint64(n.bits)), nil + case Float64Type: + return AppendFloat64(b, math.Float64frombits(n.bits)), nil + case Float32Type: + return AppendFloat32(b, math.Float32frombits(uint32(n.bits))), nil + default: + return AppendInt64(b, 0), nil + } +} + +// EncodeMsg implements msgp.Encodable +func (n *Number) EncodeMsg(w *Writer) error { + switch n.typ { + case IntType: + return w.WriteInt64(int64(n.bits)) + case UintType: + return w.WriteUint64(n.bits) + case Float64Type: + return w.WriteFloat64(math.Float64frombits(n.bits)) + case Float32Type: + return w.WriteFloat32(math.Float32frombits(uint32(n.bits))) + default: + return w.WriteInt64(0) + } +} + +// Msgsize implements msgp.Sizer +func (n *Number) Msgsize() int { + switch n.typ { + case Float32Type: + return Float32Size + case Float64Type: + return Float64Size + case IntType: + return Int64Size + case UintType: + return Uint64Size + default: + return 1 // fixint(0) + } +} + +// MarshalJSON implements json.Marshaler +func (n *Number) MarshalJSON() ([]byte, error) { + t := n.Type() + if t == InvalidType { + return []byte{'0'}, nil + } + out := make([]byte, 0, 32) + switch t { + case Float32Type, Float64Type: + f, _ := n.Float() + return strconv.AppendFloat(out, f, 'f', -1, 64), nil + case IntType: + i, _ := n.Int() + return strconv.AppendInt(out, i, 10), nil + case UintType: + u, _ := n.Uint() + return strconv.AppendUint(out, u, 10), nil + default: + panic("(*Number).typ is invalid") + } +} + +// String implements fmt.Stringer +func (n *Number) String() string { + switch n.typ { + case InvalidType: + return "0" + case Float32Type, Float64Type: + f, _ := n.Float() + return strconv.FormatFloat(f, 'f', -1, 64) + case IntType: + i, _ := n.Int() + return strconv.FormatInt(i, 10) + case UintType: + u, _ := n.Uint() + return strconv.FormatUint(u, 10) + default: + panic("(*Number).typ is invalid") + } +} diff --git a/vendor/github.com/tinylib/msgp/msgp/purego.go b/vendor/github.com/tinylib/msgp/msgp/purego.go new file mode 100644 index 00000000..c828f7ec --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/purego.go @@ -0,0 +1,15 @@ +// +build purego appengine + +package msgp + +// let's just assume appengine +// uses 64-bit hardware... +const smallint = false + +func UnsafeString(b []byte) string { + return string(b) +} + +func UnsafeBytes(s string) []byte { + return []byte(s) +} diff --git a/vendor/github.com/tinylib/msgp/msgp/read.go b/vendor/github.com/tinylib/msgp/msgp/read.go new file mode 100644 index 00000000..aa668c57 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/read.go @@ -0,0 +1,1358 @@ +package msgp + +import ( + "io" + "math" + "sync" + "time" + + "github.com/philhofer/fwd" +) + +// where we keep old *Readers +var readerPool = sync.Pool{New: func() interface{} { return &Reader{} }} + +// Type is a MessagePack wire type, +// including this package's built-in +// extension types. +type Type byte + +// MessagePack Types +// +// The zero value of Type +// is InvalidType. +const ( + InvalidType Type = iota + + // MessagePack built-in types + + StrType + BinType + MapType + ArrayType + Float64Type + Float32Type + BoolType + IntType + UintType + NilType + ExtensionType + + // pseudo-types provided + // by extensions + + Complex64Type + Complex128Type + TimeType + + _maxtype +) + +// String implements fmt.Stringer +func (t Type) String() string { + switch t { + case StrType: + return "str" + case BinType: + return "bin" + case MapType: + return "map" + case ArrayType: + return "array" + case Float64Type: + return "float64" + case Float32Type: + return "float32" + case BoolType: + return "bool" + case UintType: + return "uint" + case IntType: + return "int" + case ExtensionType: + return "ext" + case NilType: + return "nil" + default: + return "" + } +} + +func freeR(m *Reader) { + readerPool.Put(m) +} + +// Unmarshaler is the interface fulfilled +// by objects that know how to unmarshal +// themselves from MessagePack. +// UnmarshalMsg unmarshals the object +// from binary, returing any leftover +// bytes and any errors encountered. +type Unmarshaler interface { + UnmarshalMsg([]byte) ([]byte, error) +} + +// Decodable is the interface fulfilled +// by objects that know how to read +// themselves from a *Reader. +type Decodable interface { + DecodeMsg(*Reader) error +} + +// Decode decodes 'd' from 'r'. +func Decode(r io.Reader, d Decodable) error { + rd := NewReader(r) + err := d.DecodeMsg(rd) + freeR(rd) + return err +} + +// NewReader returns a *Reader that +// reads from the provided reader. The +// reader will be buffered. +func NewReader(r io.Reader) *Reader { + p := readerPool.Get().(*Reader) + if p.R == nil { + p.R = fwd.NewReader(r) + } else { + p.R.Reset(r) + } + return p +} + +// NewReaderSize returns a *Reader with a buffer of the given size. +// (This is vastly preferable to passing the decoder a reader that is already buffered.) +func NewReaderSize(r io.Reader, sz int) *Reader { + return &Reader{R: fwd.NewReaderSize(r, sz)} +} + +// Reader wraps an io.Reader and provides +// methods to read MessagePack-encoded values +// from it. Readers are buffered. +type Reader struct { + // R is the buffered reader + // that the Reader uses + // to decode MessagePack. + // The Reader itself + // is stateless; all the + // buffering is done + // within R. + R *fwd.Reader + scratch []byte +} + +// Read implements `io.Reader` +func (m *Reader) Read(p []byte) (int, error) { + return m.R.Read(p) +} + +// CopyNext reads the next object from m without decoding it and writes it to w. +// It avoids unnecessary copies internally. +func (m *Reader) CopyNext(w io.Writer) (int64, error) { + sz, o, err := getNextSize(m.R) + if err != nil { + return 0, err + } + + var n int64 + // Opportunistic optimization: if we can fit the whole thing in the m.R + // buffer, then just get a pointer to that, and pass it to w.Write, + // avoiding an allocation. + if int(sz) <= m.R.BufferSize() { + var nn int + var buf []byte + buf, err = m.R.Next(int(sz)) + if err != nil { + if err == io.ErrUnexpectedEOF { + err = ErrShortBytes + } + return 0, err + } + nn, err = w.Write(buf) + n += int64(nn) + } else { + // Fall back to io.CopyN. + // May avoid allocating if w is a ReaderFrom (e.g. bytes.Buffer) + n, err = io.CopyN(w, m.R, int64(sz)) + if err == io.ErrUnexpectedEOF { + err = ErrShortBytes + } + } + if err != nil { + return n, err + } else if n < int64(sz) { + return n, io.ErrShortWrite + } + + // for maps and slices, read elements + for x := uintptr(0); x < o; x++ { + var n2 int64 + n2, err = m.CopyNext(w) + if err != nil { + return n, err + } + n += n2 + } + return n, nil +} + +// ReadFull implements `io.ReadFull` +func (m *Reader) ReadFull(p []byte) (int, error) { + return m.R.ReadFull(p) +} + +// Reset resets the underlying reader. +func (m *Reader) Reset(r io.Reader) { m.R.Reset(r) } + +// Buffered returns the number of bytes currently in the read buffer. +func (m *Reader) Buffered() int { return m.R.Buffered() } + +// BufferSize returns the capacity of the read buffer. +func (m *Reader) BufferSize() int { return m.R.BufferSize() } + +// NextType returns the next object type to be decoded. +func (m *Reader) NextType() (Type, error) { + p, err := m.R.Peek(1) + if err != nil { + return InvalidType, err + } + t := getType(p[0]) + if t == InvalidType { + return t, InvalidPrefixError(p[0]) + } + if t == ExtensionType { + v, err := m.peekExtensionType() + if err != nil { + return InvalidType, err + } + switch v { + case Complex64Extension: + return Complex64Type, nil + case Complex128Extension: + return Complex128Type, nil + case TimeExtension: + return TimeType, nil + } + } + return t, nil +} + +// IsNil returns whether or not +// the next byte is a null messagepack byte +func (m *Reader) IsNil() bool { + p, err := m.R.Peek(1) + return err == nil && p[0] == mnil +} + +// getNextSize returns the size of the next object on the wire. +// returns (obj size, obj elements, error) +// only maps and arrays have non-zero obj elements +// for maps and arrays, obj size does not include elements +// +// use uintptr b/c it's guaranteed to be large enough +// to hold whatever we can fit in memory. +func getNextSize(r *fwd.Reader) (uintptr, uintptr, error) { + b, err := r.Peek(1) + if err != nil { + return 0, 0, err + } + lead := b[0] + spec := &sizes[lead] + size, mode := spec.size, spec.extra + if size == 0 { + return 0, 0, InvalidPrefixError(lead) + } + if mode >= 0 { + return uintptr(size), uintptr(mode), nil + } + b, err = r.Peek(int(size)) + if err != nil { + return 0, 0, err + } + switch mode { + case extra8: + return uintptr(size) + uintptr(b[1]), 0, nil + case extra16: + return uintptr(size) + uintptr(big.Uint16(b[1:])), 0, nil + case extra32: + return uintptr(size) + uintptr(big.Uint32(b[1:])), 0, nil + case map16v: + return uintptr(size), 2 * uintptr(big.Uint16(b[1:])), nil + case map32v: + return uintptr(size), 2 * uintptr(big.Uint32(b[1:])), nil + case array16v: + return uintptr(size), uintptr(big.Uint16(b[1:])), nil + case array32v: + return uintptr(size), uintptr(big.Uint32(b[1:])), nil + default: + return 0, 0, fatal + } +} + +// Skip skips over the next object, regardless of +// its type. If it is an array or map, the whole array +// or map will be skipped. +func (m *Reader) Skip() error { + var ( + v uintptr // bytes + o uintptr // objects + err error + p []byte + ) + + // we can use the faster + // method if we have enough + // buffered data + if m.R.Buffered() >= 5 { + p, err = m.R.Peek(5) + if err != nil { + return err + } + v, o, err = getSize(p) + if err != nil { + return err + } + } else { + v, o, err = getNextSize(m.R) + if err != nil { + return err + } + } + + // 'v' is always non-zero + // if err == nil + _, err = m.R.Skip(int(v)) + if err != nil { + return err + } + + // for maps and slices, skip elements + for x := uintptr(0); x < o; x++ { + err = m.Skip() + if err != nil { + return err + } + } + return nil +} + +// ReadMapHeader reads the next object +// as a map header and returns the size +// of the map and the number of bytes written. +// It will return a TypeError{} if the next +// object is not a map. +func (m *Reader) ReadMapHeader() (sz uint32, err error) { + var p []byte + var lead byte + p, err = m.R.Peek(1) + if err != nil { + return + } + lead = p[0] + if isfixmap(lead) { + sz = uint32(rfixmap(lead)) + _, err = m.R.Skip(1) + return + } + switch lead { + case mmap16: + p, err = m.R.Next(3) + if err != nil { + return + } + sz = uint32(big.Uint16(p[1:])) + return + case mmap32: + p, err = m.R.Next(5) + if err != nil { + return + } + sz = big.Uint32(p[1:]) + return + default: + err = badPrefix(MapType, lead) + return + } +} + +// ReadMapKey reads either a 'str' or 'bin' field from +// the reader and returns the value as a []byte. It uses +// scratch for storage if it is large enough. +func (m *Reader) ReadMapKey(scratch []byte) ([]byte, error) { + out, err := m.ReadStringAsBytes(scratch) + if err != nil { + if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { + return m.ReadBytes(scratch) + } + return nil, err + } + return out, nil +} + +// MapKeyPtr returns a []byte pointing to the contents +// of a valid map key. The key cannot be empty, and it +// must be shorter than the total buffer size of the +// *Reader. Additionally, the returned slice is only +// valid until the next *Reader method call. Users +// should exercise extreme care when using this +// method; writing into the returned slice may +// corrupt future reads. +func (m *Reader) ReadMapKeyPtr() ([]byte, error) { + p, err := m.R.Peek(1) + if err != nil { + return nil, err + } + lead := p[0] + var read int + if isfixstr(lead) { + read = int(rfixstr(lead)) + m.R.Skip(1) + goto fill + } + switch lead { + case mstr8, mbin8: + p, err = m.R.Next(2) + if err != nil { + return nil, err + } + read = int(p[1]) + case mstr16, mbin16: + p, err = m.R.Next(3) + if err != nil { + return nil, err + } + read = int(big.Uint16(p[1:])) + case mstr32, mbin32: + p, err = m.R.Next(5) + if err != nil { + return nil, err + } + read = int(big.Uint32(p[1:])) + default: + return nil, badPrefix(StrType, lead) + } +fill: + if read == 0 { + return nil, ErrShortBytes + } + return m.R.Next(read) +} + +// ReadArrayHeader reads the next object as an +// array header and returns the size of the array +// and the number of bytes read. +func (m *Reader) ReadArrayHeader() (sz uint32, err error) { + var lead byte + var p []byte + p, err = m.R.Peek(1) + if err != nil { + return + } + lead = p[0] + if isfixarray(lead) { + sz = uint32(rfixarray(lead)) + _, err = m.R.Skip(1) + return + } + switch lead { + case marray16: + p, err = m.R.Next(3) + if err != nil { + return + } + sz = uint32(big.Uint16(p[1:])) + return + + case marray32: + p, err = m.R.Next(5) + if err != nil { + return + } + sz = big.Uint32(p[1:]) + return + + default: + err = badPrefix(ArrayType, lead) + return + } +} + +// ReadNil reads a 'nil' MessagePack byte from the reader +func (m *Reader) ReadNil() error { + p, err := m.R.Peek(1) + if err != nil { + return err + } + if p[0] != mnil { + return badPrefix(NilType, p[0]) + } + _, err = m.R.Skip(1) + return err +} + +// ReadFloat64 reads a float64 from the reader. +// (If the value on the wire is encoded as a float32, +// it will be up-cast to a float64.) +func (m *Reader) ReadFloat64() (f float64, err error) { + var p []byte + p, err = m.R.Peek(9) + if err != nil { + // we'll allow a coversion from float32 to float64, + // since we don't lose any precision + if err == io.EOF && len(p) > 0 && p[0] == mfloat32 { + ef, err := m.ReadFloat32() + return float64(ef), err + } + return + } + if p[0] != mfloat64 { + // see above + if p[0] == mfloat32 { + ef, err := m.ReadFloat32() + return float64(ef), err + } + err = badPrefix(Float64Type, p[0]) + return + } + f = math.Float64frombits(getMuint64(p)) + _, err = m.R.Skip(9) + return +} + +// ReadFloat32 reads a float32 from the reader +func (m *Reader) ReadFloat32() (f float32, err error) { + var p []byte + p, err = m.R.Peek(5) + if err != nil { + return + } + if p[0] != mfloat32 { + err = badPrefix(Float32Type, p[0]) + return + } + f = math.Float32frombits(getMuint32(p)) + _, err = m.R.Skip(5) + return +} + +// ReadBool reads a bool from the reader +func (m *Reader) ReadBool() (b bool, err error) { + var p []byte + p, err = m.R.Peek(1) + if err != nil { + return + } + switch p[0] { + case mtrue: + b = true + case mfalse: + default: + err = badPrefix(BoolType, p[0]) + return + } + _, err = m.R.Skip(1) + return +} + +// ReadInt64 reads an int64 from the reader +func (m *Reader) ReadInt64() (i int64, err error) { + var p []byte + var lead byte + p, err = m.R.Peek(1) + if err != nil { + return + } + lead = p[0] + + if isfixint(lead) { + i = int64(rfixint(lead)) + _, err = m.R.Skip(1) + return + } else if isnfixint(lead) { + i = int64(rnfixint(lead)) + _, err = m.R.Skip(1) + return + } + + switch lead { + case mint8: + p, err = m.R.Next(2) + if err != nil { + return + } + i = int64(getMint8(p)) + return + + case muint8: + p, err = m.R.Next(2) + if err != nil { + return + } + i = int64(getMuint8(p)) + return + + case mint16: + p, err = m.R.Next(3) + if err != nil { + return + } + i = int64(getMint16(p)) + return + + case muint16: + p, err = m.R.Next(3) + if err != nil { + return + } + i = int64(getMuint16(p)) + return + + case mint32: + p, err = m.R.Next(5) + if err != nil { + return + } + i = int64(getMint32(p)) + return + + case muint32: + p, err = m.R.Next(5) + if err != nil { + return + } + i = int64(getMuint32(p)) + return + + case mint64: + p, err = m.R.Next(9) + if err != nil { + return + } + i = getMint64(p) + return + + case muint64: + p, err = m.R.Next(9) + if err != nil { + return + } + u := getMuint64(p) + if u > math.MaxInt64 { + err = UintOverflow{Value: u, FailedBitsize: 64} + return + } + i = int64(u) + return + + default: + err = badPrefix(IntType, lead) + return + } +} + +// ReadInt32 reads an int32 from the reader +func (m *Reader) ReadInt32() (i int32, err error) { + var in int64 + in, err = m.ReadInt64() + if in > math.MaxInt32 || in < math.MinInt32 { + err = IntOverflow{Value: in, FailedBitsize: 32} + return + } + i = int32(in) + return +} + +// ReadInt16 reads an int16 from the reader +func (m *Reader) ReadInt16() (i int16, err error) { + var in int64 + in, err = m.ReadInt64() + if in > math.MaxInt16 || in < math.MinInt16 { + err = IntOverflow{Value: in, FailedBitsize: 16} + return + } + i = int16(in) + return +} + +// ReadInt8 reads an int8 from the reader +func (m *Reader) ReadInt8() (i int8, err error) { + var in int64 + in, err = m.ReadInt64() + if in > math.MaxInt8 || in < math.MinInt8 { + err = IntOverflow{Value: in, FailedBitsize: 8} + return + } + i = int8(in) + return +} + +// ReadInt reads an int from the reader +func (m *Reader) ReadInt() (i int, err error) { + if smallint { + var in int32 + in, err = m.ReadInt32() + i = int(in) + return + } + var in int64 + in, err = m.ReadInt64() + i = int(in) + return +} + +// ReadUint64 reads a uint64 from the reader +func (m *Reader) ReadUint64() (u uint64, err error) { + var p []byte + var lead byte + p, err = m.R.Peek(1) + if err != nil { + return + } + lead = p[0] + if isfixint(lead) { + u = uint64(rfixint(lead)) + _, err = m.R.Skip(1) + return + } + switch lead { + case mint8: + p, err = m.R.Next(2) + if err != nil { + return + } + v := int64(getMint8(p)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + return + + case muint8: + p, err = m.R.Next(2) + if err != nil { + return + } + u = uint64(getMuint8(p)) + return + + case mint16: + p, err = m.R.Next(3) + if err != nil { + return + } + v := int64(getMint16(p)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + return + + case muint16: + p, err = m.R.Next(3) + if err != nil { + return + } + u = uint64(getMuint16(p)) + return + + case mint32: + p, err = m.R.Next(5) + if err != nil { + return + } + v := int64(getMint32(p)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + return + + case muint32: + p, err = m.R.Next(5) + if err != nil { + return + } + u = uint64(getMuint32(p)) + return + + case mint64: + p, err = m.R.Next(9) + if err != nil { + return + } + v := int64(getMint64(p)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + return + + case muint64: + p, err = m.R.Next(9) + if err != nil { + return + } + u = getMuint64(p) + return + + default: + if isnfixint(lead) { + err = UintBelowZero{Value: int64(rnfixint(lead))} + } else { + err = badPrefix(UintType, lead) + } + return + + } +} + +// ReadUint32 reads a uint32 from the reader +func (m *Reader) ReadUint32() (u uint32, err error) { + var in uint64 + in, err = m.ReadUint64() + if in > math.MaxUint32 { + err = UintOverflow{Value: in, FailedBitsize: 32} + return + } + u = uint32(in) + return +} + +// ReadUint16 reads a uint16 from the reader +func (m *Reader) ReadUint16() (u uint16, err error) { + var in uint64 + in, err = m.ReadUint64() + if in > math.MaxUint16 { + err = UintOverflow{Value: in, FailedBitsize: 16} + return + } + u = uint16(in) + return +} + +// ReadUint8 reads a uint8 from the reader +func (m *Reader) ReadUint8() (u uint8, err error) { + var in uint64 + in, err = m.ReadUint64() + if in > math.MaxUint8 { + err = UintOverflow{Value: in, FailedBitsize: 8} + return + } + u = uint8(in) + return +} + +// ReadUint reads a uint from the reader +func (m *Reader) ReadUint() (u uint, err error) { + if smallint { + var un uint32 + un, err = m.ReadUint32() + u = uint(un) + return + } + var un uint64 + un, err = m.ReadUint64() + u = uint(un) + return +} + +// ReadByte is analogous to ReadUint8. +// +// NOTE: this is *not* an implementation +// of io.ByteReader. +func (m *Reader) ReadByte() (b byte, err error) { + var in uint64 + in, err = m.ReadUint64() + if in > math.MaxUint8 { + err = UintOverflow{Value: in, FailedBitsize: 8} + return + } + b = byte(in) + return +} + +// ReadBytes reads a MessagePack 'bin' object +// from the reader and returns its value. It may +// use 'scratch' for storage if it is non-nil. +func (m *Reader) ReadBytes(scratch []byte) (b []byte, err error) { + var p []byte + var lead byte + p, err = m.R.Peek(2) + if err != nil { + return + } + lead = p[0] + var read int64 + switch lead { + case mbin8: + read = int64(p[1]) + m.R.Skip(2) + case mbin16: + p, err = m.R.Next(3) + if err != nil { + return + } + read = int64(big.Uint16(p[1:])) + case mbin32: + p, err = m.R.Next(5) + if err != nil { + return + } + read = int64(big.Uint32(p[1:])) + default: + err = badPrefix(BinType, lead) + return + } + if int64(cap(scratch)) < read { + b = make([]byte, read) + } else { + b = scratch[0:read] + } + _, err = m.R.ReadFull(b) + return +} + +// ReadBytesHeader reads the size header +// of a MessagePack 'bin' object. The user +// is responsible for dealing with the next +// 'sz' bytes from the reader in an application-specific +// way. +func (m *Reader) ReadBytesHeader() (sz uint32, err error) { + var p []byte + p, err = m.R.Peek(1) + if err != nil { + return + } + switch p[0] { + case mbin8: + p, err = m.R.Next(2) + if err != nil { + return + } + sz = uint32(p[1]) + return + case mbin16: + p, err = m.R.Next(3) + if err != nil { + return + } + sz = uint32(big.Uint16(p[1:])) + return + case mbin32: + p, err = m.R.Next(5) + if err != nil { + return + } + sz = uint32(big.Uint32(p[1:])) + return + default: + err = badPrefix(BinType, p[0]) + return + } +} + +// ReadExactBytes reads a MessagePack 'bin'-encoded +// object off of the wire into the provided slice. An +// ArrayError will be returned if the object is not +// exactly the length of the input slice. +func (m *Reader) ReadExactBytes(into []byte) error { + p, err := m.R.Peek(2) + if err != nil { + return err + } + lead := p[0] + var read int64 // bytes to read + var skip int // prefix size to skip + switch lead { + case mbin8: + read = int64(p[1]) + skip = 2 + case mbin16: + p, err = m.R.Peek(3) + if err != nil { + return err + } + read = int64(big.Uint16(p[1:])) + skip = 3 + case mbin32: + p, err = m.R.Peek(5) + if err != nil { + return err + } + read = int64(big.Uint32(p[1:])) + skip = 5 + default: + return badPrefix(BinType, lead) + } + if read != int64(len(into)) { + return ArrayError{Wanted: uint32(len(into)), Got: uint32(read)} + } + m.R.Skip(skip) + _, err = m.R.ReadFull(into) + return err +} + +// ReadStringAsBytes reads a MessagePack 'str' (utf-8) string +// and returns its value as bytes. It may use 'scratch' for storage +// if it is non-nil. +func (m *Reader) ReadStringAsBytes(scratch []byte) (b []byte, err error) { + var p []byte + var lead byte + p, err = m.R.Peek(1) + if err != nil { + return + } + lead = p[0] + var read int64 + + if isfixstr(lead) { + read = int64(rfixstr(lead)) + m.R.Skip(1) + goto fill + } + + switch lead { + case mstr8: + p, err = m.R.Next(2) + if err != nil { + return + } + read = int64(uint8(p[1])) + case mstr16: + p, err = m.R.Next(3) + if err != nil { + return + } + read = int64(big.Uint16(p[1:])) + case mstr32: + p, err = m.R.Next(5) + if err != nil { + return + } + read = int64(big.Uint32(p[1:])) + default: + err = badPrefix(StrType, lead) + return + } +fill: + if int64(cap(scratch)) < read { + b = make([]byte, read) + } else { + b = scratch[0:read] + } + _, err = m.R.ReadFull(b) + return +} + +// ReadStringHeader reads a string header +// off of the wire. The user is then responsible +// for dealing with the next 'sz' bytes from +// the reader in an application-specific manner. +func (m *Reader) ReadStringHeader() (sz uint32, err error) { + var p []byte + p, err = m.R.Peek(1) + if err != nil { + return + } + lead := p[0] + if isfixstr(lead) { + sz = uint32(rfixstr(lead)) + m.R.Skip(1) + return + } + switch lead { + case mstr8: + p, err = m.R.Next(2) + if err != nil { + return + } + sz = uint32(p[1]) + return + case mstr16: + p, err = m.R.Next(3) + if err != nil { + return + } + sz = uint32(big.Uint16(p[1:])) + return + case mstr32: + p, err = m.R.Next(5) + if err != nil { + return + } + sz = big.Uint32(p[1:]) + return + default: + err = badPrefix(StrType, lead) + return + } +} + +// ReadString reads a utf-8 string from the reader +func (m *Reader) ReadString() (s string, err error) { + var p []byte + var lead byte + var read int64 + p, err = m.R.Peek(1) + if err != nil { + return + } + lead = p[0] + + if isfixstr(lead) { + read = int64(rfixstr(lead)) + m.R.Skip(1) + goto fill + } + + switch lead { + case mstr8: + p, err = m.R.Next(2) + if err != nil { + return + } + read = int64(uint8(p[1])) + case mstr16: + p, err = m.R.Next(3) + if err != nil { + return + } + read = int64(big.Uint16(p[1:])) + case mstr32: + p, err = m.R.Next(5) + if err != nil { + return + } + read = int64(big.Uint32(p[1:])) + default: + err = badPrefix(StrType, lead) + return + } +fill: + if read == 0 { + s, err = "", nil + return + } + // reading into the memory + // that will become the string + // itself has vastly superior + // worst-case performance, because + // the reader buffer doesn't have + // to be large enough to hold the string. + // the idea here is to make it more + // difficult for someone malicious + // to cause the system to run out of + // memory by sending very large strings. + // + // NOTE: this works because the argument + // passed to (*fwd.Reader).ReadFull escapes + // to the heap; its argument may, in turn, + // be passed to the underlying reader, and + // thus escape analysis *must* conclude that + // 'out' escapes. + out := make([]byte, read) + _, err = m.R.ReadFull(out) + if err != nil { + return + } + s = UnsafeString(out) + return +} + +// ReadComplex64 reads a complex64 from the reader +func (m *Reader) ReadComplex64() (f complex64, err error) { + var p []byte + p, err = m.R.Peek(10) + if err != nil { + return + } + if p[0] != mfixext8 { + err = badPrefix(Complex64Type, p[0]) + return + } + if int8(p[1]) != Complex64Extension { + err = errExt(int8(p[1]), Complex64Extension) + return + } + f = complex(math.Float32frombits(big.Uint32(p[2:])), + math.Float32frombits(big.Uint32(p[6:]))) + _, err = m.R.Skip(10) + return +} + +// ReadComplex128 reads a complex128 from the reader +func (m *Reader) ReadComplex128() (f complex128, err error) { + var p []byte + p, err = m.R.Peek(18) + if err != nil { + return + } + if p[0] != mfixext16 { + err = badPrefix(Complex128Type, p[0]) + return + } + if int8(p[1]) != Complex128Extension { + err = errExt(int8(p[1]), Complex128Extension) + return + } + f = complex(math.Float64frombits(big.Uint64(p[2:])), + math.Float64frombits(big.Uint64(p[10:]))) + _, err = m.R.Skip(18) + return +} + +// ReadMapStrIntf reads a MessagePack map into a map[string]interface{}. +// (You must pass a non-nil map into the function.) +func (m *Reader) ReadMapStrIntf(mp map[string]interface{}) (err error) { + var sz uint32 + sz, err = m.ReadMapHeader() + if err != nil { + return + } + for key := range mp { + delete(mp, key) + } + for i := uint32(0); i < sz; i++ { + var key string + var val interface{} + key, err = m.ReadString() + if err != nil { + return + } + val, err = m.ReadIntf() + if err != nil { + return + } + mp[key] = val + } + return +} + +// ReadTime reads a time.Time object from the reader. +// The returned time's location will be set to time.Local. +func (m *Reader) ReadTime() (t time.Time, err error) { + var p []byte + p, err = m.R.Peek(15) + if err != nil { + return + } + if p[0] != mext8 || p[1] != 12 { + err = badPrefix(TimeType, p[0]) + return + } + if int8(p[2]) != TimeExtension { + err = errExt(int8(p[2]), TimeExtension) + return + } + sec, nsec := getUnix(p[3:]) + t = time.Unix(sec, int64(nsec)).Local() + _, err = m.R.Skip(15) + return +} + +// ReadIntf reads out the next object as a raw interface{}. +// Arrays are decoded as []interface{}, and maps are decoded +// as map[string]interface{}. Integers are decoded as int64 +// and unsigned integers are decoded as uint64. +func (m *Reader) ReadIntf() (i interface{}, err error) { + var t Type + t, err = m.NextType() + if err != nil { + return + } + switch t { + case BoolType: + i, err = m.ReadBool() + return + + case IntType: + i, err = m.ReadInt64() + return + + case UintType: + i, err = m.ReadUint64() + return + + case BinType: + i, err = m.ReadBytes(nil) + return + + case StrType: + i, err = m.ReadString() + return + + case Complex64Type: + i, err = m.ReadComplex64() + return + + case Complex128Type: + i, err = m.ReadComplex128() + return + + case TimeType: + i, err = m.ReadTime() + return + + case ExtensionType: + var t int8 + t, err = m.peekExtensionType() + if err != nil { + return + } + f, ok := extensionReg[t] + if ok { + e := f() + err = m.ReadExtension(e) + i = e + return + } + var e RawExtension + e.Type = t + err = m.ReadExtension(&e) + i = &e + return + + case MapType: + mp := make(map[string]interface{}) + err = m.ReadMapStrIntf(mp) + i = mp + return + + case NilType: + err = m.ReadNil() + i = nil + return + + case Float32Type: + i, err = m.ReadFloat32() + return + + case Float64Type: + i, err = m.ReadFloat64() + return + + case ArrayType: + var sz uint32 + sz, err = m.ReadArrayHeader() + + if err != nil { + return + } + out := make([]interface{}, int(sz)) + for j := range out { + out[j], err = m.ReadIntf() + if err != nil { + return + } + } + i = out + return + + default: + return nil, fatal // unreachable + } +} diff --git a/vendor/github.com/tinylib/msgp/msgp/read_bytes.go b/vendor/github.com/tinylib/msgp/msgp/read_bytes.go new file mode 100644 index 00000000..f53f84d0 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/read_bytes.go @@ -0,0 +1,1197 @@ +package msgp + +import ( + "bytes" + "encoding/binary" + "math" + "time" +) + +var big = binary.BigEndian + +// NextType returns the type of the next +// object in the slice. If the length +// of the input is zero, it returns +// InvalidType. +func NextType(b []byte) Type { + if len(b) == 0 { + return InvalidType + } + spec := sizes[b[0]] + t := spec.typ + if t == ExtensionType && len(b) > int(spec.size) { + var tp int8 + if spec.extra == constsize { + tp = int8(b[1]) + } else { + tp = int8(b[spec.size-1]) + } + switch tp { + case TimeExtension: + return TimeType + case Complex128Extension: + return Complex128Type + case Complex64Extension: + return Complex64Type + default: + return ExtensionType + } + } + return t +} + +// IsNil returns true if len(b)>0 and +// the leading byte is a 'nil' MessagePack +// byte; false otherwise +func IsNil(b []byte) bool { + if len(b) != 0 && b[0] == mnil { + return true + } + return false +} + +// Raw is raw MessagePack. +// Raw allows you to read and write +// data without interpreting its contents. +type Raw []byte + +// MarshalMsg implements msgp.Marshaler. +// It appends the raw contents of 'raw' +// to the provided byte slice. If 'raw' +// is 0 bytes, 'nil' will be appended instead. +func (r Raw) MarshalMsg(b []byte) ([]byte, error) { + i := len(r) + if i == 0 { + return AppendNil(b), nil + } + o, l := ensure(b, i) + copy(o[l:], []byte(r)) + return o, nil +} + +// UnmarshalMsg implements msgp.Unmarshaler. +// It sets the contents of *Raw to be the next +// object in the provided byte slice. +func (r *Raw) UnmarshalMsg(b []byte) ([]byte, error) { + l := len(b) + out, err := Skip(b) + if err != nil { + return b, err + } + rlen := l - len(out) + if IsNil(b[:rlen]) { + rlen = 0 + } + if cap(*r) < rlen { + *r = make(Raw, rlen) + } else { + *r = (*r)[0:rlen] + } + copy(*r, b[:rlen]) + return out, nil +} + +// EncodeMsg implements msgp.Encodable. +// It writes the raw bytes to the writer. +// If r is empty, it writes 'nil' instead. +func (r Raw) EncodeMsg(w *Writer) error { + if len(r) == 0 { + return w.WriteNil() + } + _, err := w.Write([]byte(r)) + return err +} + +// DecodeMsg implements msgp.Decodable. +// It sets the value of *Raw to be the +// next object on the wire. +func (r *Raw) DecodeMsg(f *Reader) error { + *r = (*r)[:0] + err := appendNext(f, (*[]byte)(r)) + if IsNil(*r) { + *r = (*r)[:0] + } + return err +} + +// Msgsize implements msgp.Sizer +func (r Raw) Msgsize() int { + l := len(r) + if l == 0 { + return 1 // for 'nil' + } + return l +} + +func appendNext(f *Reader, d *[]byte) error { + amt, o, err := getNextSize(f.R) + if err != nil { + return err + } + var i int + *d, i = ensure(*d, int(amt)) + _, err = f.R.ReadFull((*d)[i:]) + if err != nil { + return err + } + for o > 0 { + err = appendNext(f, d) + if err != nil { + return err + } + o-- + } + return nil +} + +// MarshalJSON implements json.Marshaler +func (r *Raw) MarshalJSON() ([]byte, error) { + var buf bytes.Buffer + _, err := UnmarshalAsJSON(&buf, []byte(*r)) + return buf.Bytes(), err +} + +// ReadMapHeaderBytes reads a map header size +// from 'b' and returns the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a map) +func ReadMapHeaderBytes(b []byte) (sz uint32, o []byte, err error) { + l := len(b) + if l < 1 { + err = ErrShortBytes + return + } + + lead := b[0] + if isfixmap(lead) { + sz = uint32(rfixmap(lead)) + o = b[1:] + return + } + + switch lead { + case mmap16: + if l < 3 { + err = ErrShortBytes + return + } + sz = uint32(big.Uint16(b[1:])) + o = b[3:] + return + + case mmap32: + if l < 5 { + err = ErrShortBytes + return + } + sz = big.Uint32(b[1:]) + o = b[5:] + return + + default: + err = badPrefix(MapType, lead) + return + } +} + +// ReadMapKeyZC attempts to read a map key +// from 'b' and returns the key bytes and the remaining bytes +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a str or bin) +func ReadMapKeyZC(b []byte) ([]byte, []byte, error) { + o, b, err := ReadStringZC(b) + if err != nil { + if tperr, ok := err.(TypeError); ok && tperr.Encoded == BinType { + return ReadBytesZC(b) + } + return nil, b, err + } + return o, b, nil +} + +// ReadArrayHeaderBytes attempts to read +// the array header size off of 'b' and return +// the size and remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not an array) +func ReadArrayHeaderBytes(b []byte) (sz uint32, o []byte, err error) { + if len(b) < 1 { + return 0, nil, ErrShortBytes + } + lead := b[0] + if isfixarray(lead) { + sz = uint32(rfixarray(lead)) + o = b[1:] + return + } + + switch lead { + case marray16: + if len(b) < 3 { + err = ErrShortBytes + return + } + sz = uint32(big.Uint16(b[1:])) + o = b[3:] + return + + case marray32: + if len(b) < 5 { + err = ErrShortBytes + return + } + sz = big.Uint32(b[1:]) + o = b[5:] + return + + default: + err = badPrefix(ArrayType, lead) + return + } +} + +// ReadNilBytes tries to read a "nil" byte +// off of 'b' and return the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a 'nil') +// - InvalidPrefixError +func ReadNilBytes(b []byte) ([]byte, error) { + if len(b) < 1 { + return nil, ErrShortBytes + } + if b[0] != mnil { + return b, badPrefix(NilType, b[0]) + } + return b[1:], nil +} + +// ReadFloat64Bytes tries to read a float64 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a float64) +func ReadFloat64Bytes(b []byte) (f float64, o []byte, err error) { + if len(b) < 9 { + if len(b) >= 5 && b[0] == mfloat32 { + var tf float32 + tf, o, err = ReadFloat32Bytes(b) + f = float64(tf) + return + } + err = ErrShortBytes + return + } + + if b[0] != mfloat64 { + if b[0] == mfloat32 { + var tf float32 + tf, o, err = ReadFloat32Bytes(b) + f = float64(tf) + return + } + err = badPrefix(Float64Type, b[0]) + return + } + + f = math.Float64frombits(getMuint64(b)) + o = b[9:] + return +} + +// ReadFloat32Bytes tries to read a float64 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a float32) +func ReadFloat32Bytes(b []byte) (f float32, o []byte, err error) { + if len(b) < 5 { + err = ErrShortBytes + return + } + + if b[0] != mfloat32 { + err = TypeError{Method: Float32Type, Encoded: getType(b[0])} + return + } + + f = math.Float32frombits(getMuint32(b)) + o = b[5:] + return +} + +// ReadBoolBytes tries to read a float64 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a bool) +func ReadBoolBytes(b []byte) (bool, []byte, error) { + if len(b) < 1 { + return false, b, ErrShortBytes + } + switch b[0] { + case mtrue: + return true, b[1:], nil + case mfalse: + return false, b[1:], nil + default: + return false, b, badPrefix(BoolType, b[0]) + } +} + +// ReadInt64Bytes tries to read an int64 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError (not a int) +func ReadInt64Bytes(b []byte) (i int64, o []byte, err error) { + l := len(b) + if l < 1 { + return 0, nil, ErrShortBytes + } + + lead := b[0] + if isfixint(lead) { + i = int64(rfixint(lead)) + o = b[1:] + return + } + if isnfixint(lead) { + i = int64(rnfixint(lead)) + o = b[1:] + return + } + + switch lead { + case mint8: + if l < 2 { + err = ErrShortBytes + return + } + i = int64(getMint8(b)) + o = b[2:] + return + + case muint8: + if l < 2 { + err = ErrShortBytes + return + } + i = int64(getMuint8(b)) + o = b[2:] + return + + case mint16: + if l < 3 { + err = ErrShortBytes + return + } + i = int64(getMint16(b)) + o = b[3:] + return + + case muint16: + if l < 3 { + err = ErrShortBytes + return + } + i = int64(getMuint16(b)) + o = b[3:] + return + + case mint32: + if l < 5 { + err = ErrShortBytes + return + } + i = int64(getMint32(b)) + o = b[5:] + return + + case muint32: + if l < 5 { + err = ErrShortBytes + return + } + i = int64(getMuint32(b)) + o = b[5:] + return + + case mint64: + if l < 9 { + err = ErrShortBytes + return + } + i = int64(getMint64(b)) + o = b[9:] + return + + case muint64: + if l < 9 { + err = ErrShortBytes + return + } + u := getMuint64(b) + if u > math.MaxInt64 { + err = UintOverflow{Value: u, FailedBitsize: 64} + return + } + i = int64(u) + o = b[9:] + return + + default: + err = badPrefix(IntType, lead) + return + } +} + +// ReadInt32Bytes tries to read an int32 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a int) +// - IntOverflow{} (value doesn't fit in int32) +func ReadInt32Bytes(b []byte) (int32, []byte, error) { + i, o, err := ReadInt64Bytes(b) + if i > math.MaxInt32 || i < math.MinInt32 { + return 0, o, IntOverflow{Value: i, FailedBitsize: 32} + } + return int32(i), o, err +} + +// ReadInt16Bytes tries to read an int16 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a int) +// - IntOverflow{} (value doesn't fit in int16) +func ReadInt16Bytes(b []byte) (int16, []byte, error) { + i, o, err := ReadInt64Bytes(b) + if i > math.MaxInt16 || i < math.MinInt16 { + return 0, o, IntOverflow{Value: i, FailedBitsize: 16} + } + return int16(i), o, err +} + +// ReadInt8Bytes tries to read an int16 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a int) +// - IntOverflow{} (value doesn't fit in int8) +func ReadInt8Bytes(b []byte) (int8, []byte, error) { + i, o, err := ReadInt64Bytes(b) + if i > math.MaxInt8 || i < math.MinInt8 { + return 0, o, IntOverflow{Value: i, FailedBitsize: 8} + } + return int8(i), o, err +} + +// ReadIntBytes tries to read an int +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a int) +// - IntOverflow{} (value doesn't fit in int; 32-bit platforms only) +func ReadIntBytes(b []byte) (int, []byte, error) { + if smallint { + i, b, err := ReadInt32Bytes(b) + return int(i), b, err + } + i, b, err := ReadInt64Bytes(b) + return int(i), b, err +} + +// ReadUint64Bytes tries to read a uint64 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a uint) +func ReadUint64Bytes(b []byte) (u uint64, o []byte, err error) { + l := len(b) + if l < 1 { + return 0, nil, ErrShortBytes + } + + lead := b[0] + if isfixint(lead) { + u = uint64(rfixint(lead)) + o = b[1:] + return + } + + switch lead { + case mint8: + if l < 2 { + err = ErrShortBytes + return + } + v := int64(getMint8(b)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + o = b[2:] + return + + case muint8: + if l < 2 { + err = ErrShortBytes + return + } + u = uint64(getMuint8(b)) + o = b[2:] + return + + case mint16: + if l < 3 { + err = ErrShortBytes + return + } + v := int64(getMint16(b)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + o = b[3:] + return + + case muint16: + if l < 3 { + err = ErrShortBytes + return + } + u = uint64(getMuint16(b)) + o = b[3:] + return + + case mint32: + if l < 5 { + err = ErrShortBytes + return + } + v := int64(getMint32(b)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + o = b[5:] + return + + case muint32: + if l < 5 { + err = ErrShortBytes + return + } + u = uint64(getMuint32(b)) + o = b[5:] + return + + case mint64: + if l < 9 { + err = ErrShortBytes + return + } + v := int64(getMint64(b)) + if v < 0 { + err = UintBelowZero{Value: v} + return + } + u = uint64(v) + o = b[9:] + return + + case muint64: + if l < 9 { + err = ErrShortBytes + return + } + u = getMuint64(b) + o = b[9:] + return + + default: + if isnfixint(lead) { + err = UintBelowZero{Value: int64(rnfixint(lead))} + } else { + err = badPrefix(UintType, lead) + } + return + } +} + +// ReadUint32Bytes tries to read a uint32 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a uint) +// - UintOverflow{} (value too large for uint32) +func ReadUint32Bytes(b []byte) (uint32, []byte, error) { + v, o, err := ReadUint64Bytes(b) + if v > math.MaxUint32 { + return 0, nil, UintOverflow{Value: v, FailedBitsize: 32} + } + return uint32(v), o, err +} + +// ReadUint16Bytes tries to read a uint16 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a uint) +// - UintOverflow{} (value too large for uint16) +func ReadUint16Bytes(b []byte) (uint16, []byte, error) { + v, o, err := ReadUint64Bytes(b) + if v > math.MaxUint16 { + return 0, nil, UintOverflow{Value: v, FailedBitsize: 16} + } + return uint16(v), o, err +} + +// ReadUint8Bytes tries to read a uint8 +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a uint) +// - UintOverflow{} (value too large for uint8) +func ReadUint8Bytes(b []byte) (uint8, []byte, error) { + v, o, err := ReadUint64Bytes(b) + if v > math.MaxUint8 { + return 0, nil, UintOverflow{Value: v, FailedBitsize: 8} + } + return uint8(v), o, err +} + +// ReadUintBytes tries to read a uint +// from 'b' and return the value and the remaining bytes. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a uint) +// - UintOverflow{} (value too large for uint; 32-bit platforms only) +func ReadUintBytes(b []byte) (uint, []byte, error) { + if smallint { + u, b, err := ReadUint32Bytes(b) + return uint(u), b, err + } + u, b, err := ReadUint64Bytes(b) + return uint(u), b, err +} + +// ReadByteBytes is analogous to ReadUint8Bytes +func ReadByteBytes(b []byte) (byte, []byte, error) { + return ReadUint8Bytes(b) +} + +// ReadBytesBytes reads a 'bin' object +// from 'b' and returns its vaue and +// the remaining bytes in 'b'. +// Possible errors: +// - ErrShortBytes (too few bytes) +// - TypeError{} (not a 'bin' object) +func ReadBytesBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) { + return readBytesBytes(b, scratch, false) +} + +func readBytesBytes(b []byte, scratch []byte, zc bool) (v []byte, o []byte, err error) { + l := len(b) + if l < 1 { + return nil, nil, ErrShortBytes + } + + lead := b[0] + var read int + switch lead { + case mbin8: + if l < 2 { + err = ErrShortBytes + return + } + + read = int(b[1]) + b = b[2:] + + case mbin16: + if l < 3 { + err = ErrShortBytes + return + } + read = int(big.Uint16(b[1:])) + b = b[3:] + + case mbin32: + if l < 5 { + err = ErrShortBytes + return + } + read = int(big.Uint32(b[1:])) + b = b[5:] + + default: + err = badPrefix(BinType, lead) + return + } + + if len(b) < read { + err = ErrShortBytes + return + } + + // zero-copy + if zc { + v = b[0:read] + o = b[read:] + return + } + + if cap(scratch) >= read { + v = scratch[0:read] + } else { + v = make([]byte, read) + } + + o = b[copy(v, b):] + return +} + +// ReadBytesZC extracts the messagepack-encoded +// binary field without copying. The returned []byte +// points to the same memory as the input slice. +// Possible errors: +// - ErrShortBytes (b not long enough) +// - TypeError{} (object not 'bin') +func ReadBytesZC(b []byte) (v []byte, o []byte, err error) { + return readBytesBytes(b, nil, true) +} + +func ReadExactBytes(b []byte, into []byte) (o []byte, err error) { + l := len(b) + if l < 1 { + err = ErrShortBytes + return + } + + lead := b[0] + var read uint32 + var skip int + switch lead { + case mbin8: + if l < 2 { + err = ErrShortBytes + return + } + + read = uint32(b[1]) + skip = 2 + + case mbin16: + if l < 3 { + err = ErrShortBytes + return + } + read = uint32(big.Uint16(b[1:])) + skip = 3 + + case mbin32: + if l < 5 { + err = ErrShortBytes + return + } + read = uint32(big.Uint32(b[1:])) + skip = 5 + + default: + err = badPrefix(BinType, lead) + return + } + + if read != uint32(len(into)) { + err = ArrayError{Wanted: uint32(len(into)), Got: read} + return + } + + o = b[skip+copy(into, b[skip:]):] + return +} + +// ReadStringZC reads a messagepack string field +// without copying. The returned []byte points +// to the same memory as the input slice. +// Possible errors: +// - ErrShortBytes (b not long enough) +// - TypeError{} (object not 'str') +func ReadStringZC(b []byte) (v []byte, o []byte, err error) { + l := len(b) + if l < 1 { + return nil, nil, ErrShortBytes + } + + lead := b[0] + var read int + + if isfixstr(lead) { + read = int(rfixstr(lead)) + b = b[1:] + } else { + switch lead { + case mstr8: + if l < 2 { + err = ErrShortBytes + return + } + read = int(b[1]) + b = b[2:] + + case mstr16: + if l < 3 { + err = ErrShortBytes + return + } + read = int(big.Uint16(b[1:])) + b = b[3:] + + case mstr32: + if l < 5 { + err = ErrShortBytes + return + } + read = int(big.Uint32(b[1:])) + b = b[5:] + + default: + err = TypeError{Method: StrType, Encoded: getType(lead)} + return + } + } + + if len(b) < read { + err = ErrShortBytes + return + } + + v = b[0:read] + o = b[read:] + return +} + +// ReadStringBytes reads a 'str' object +// from 'b' and returns its value and the +// remaining bytes in 'b'. +// Possible errors: +// - ErrShortBytes (b not long enough) +// - TypeError{} (not 'str' type) +// - InvalidPrefixError +func ReadStringBytes(b []byte) (string, []byte, error) { + v, o, err := ReadStringZC(b) + return string(v), o, err +} + +// ReadStringAsBytes reads a 'str' object +// into a slice of bytes. 'v' is the value of +// the 'str' object, which may reside in memory +// pointed to by 'scratch.' 'o' is the remaining bytes +// in 'b.'' +// Possible errors: +// - ErrShortBytes (b not long enough) +// - TypeError{} (not 'str' type) +// - InvalidPrefixError (unknown type marker) +func ReadStringAsBytes(b []byte, scratch []byte) (v []byte, o []byte, err error) { + var tmp []byte + tmp, o, err = ReadStringZC(b) + v = append(scratch[:0], tmp...) + return +} + +// ReadComplex128Bytes reads a complex128 +// extension object from 'b' and returns the +// remaining bytes. +// Possible errors: +// - ErrShortBytes (not enough bytes in 'b') +// - TypeError{} (object not a complex128) +// - InvalidPrefixError +// - ExtensionTypeError{} (object an extension of the correct size, but not a complex128) +func ReadComplex128Bytes(b []byte) (c complex128, o []byte, err error) { + if len(b) < 18 { + err = ErrShortBytes + return + } + if b[0] != mfixext16 { + err = badPrefix(Complex128Type, b[0]) + return + } + if int8(b[1]) != Complex128Extension { + err = errExt(int8(b[1]), Complex128Extension) + return + } + c = complex(math.Float64frombits(big.Uint64(b[2:])), + math.Float64frombits(big.Uint64(b[10:]))) + o = b[18:] + return +} + +// ReadComplex64Bytes reads a complex64 +// extension object from 'b' and returns the +// remaining bytes. +// Possible errors: +// - ErrShortBytes (not enough bytes in 'b') +// - TypeError{} (object not a complex64) +// - ExtensionTypeError{} (object an extension of the correct size, but not a complex64) +func ReadComplex64Bytes(b []byte) (c complex64, o []byte, err error) { + if len(b) < 10 { + err = ErrShortBytes + return + } + if b[0] != mfixext8 { + err = badPrefix(Complex64Type, b[0]) + return + } + if b[1] != Complex64Extension { + err = errExt(int8(b[1]), Complex64Extension) + return + } + c = complex(math.Float32frombits(big.Uint32(b[2:])), + math.Float32frombits(big.Uint32(b[6:]))) + o = b[10:] + return +} + +// ReadTimeBytes reads a time.Time +// extension object from 'b' and returns the +// remaining bytes. +// Possible errors: +// - ErrShortBytes (not enough bytes in 'b') +// - TypeError{} (object not a complex64) +// - ExtensionTypeError{} (object an extension of the correct size, but not a time.Time) +func ReadTimeBytes(b []byte) (t time.Time, o []byte, err error) { + if len(b) < 15 { + err = ErrShortBytes + return + } + if b[0] != mext8 || b[1] != 12 { + err = badPrefix(TimeType, b[0]) + return + } + if int8(b[2]) != TimeExtension { + err = errExt(int8(b[2]), TimeExtension) + return + } + sec, nsec := getUnix(b[3:]) + t = time.Unix(sec, int64(nsec)).Local() + o = b[15:] + return +} + +// ReadMapStrIntfBytes reads a map[string]interface{} +// out of 'b' and returns the map and remaining bytes. +// If 'old' is non-nil, the values will be read into that map. +func ReadMapStrIntfBytes(b []byte, old map[string]interface{}) (v map[string]interface{}, o []byte, err error) { + var sz uint32 + o = b + sz, o, err = ReadMapHeaderBytes(o) + + if err != nil { + return + } + + if old != nil { + for key := range old { + delete(old, key) + } + v = old + } else { + v = make(map[string]interface{}, int(sz)) + } + + for z := uint32(0); z < sz; z++ { + if len(o) < 1 { + err = ErrShortBytes + return + } + var key []byte + key, o, err = ReadMapKeyZC(o) + if err != nil { + return + } + var val interface{} + val, o, err = ReadIntfBytes(o) + if err != nil { + return + } + v[string(key)] = val + } + return +} + +// ReadIntfBytes attempts to read +// the next object out of 'b' as a raw interface{} and +// return the remaining bytes. +func ReadIntfBytes(b []byte) (i interface{}, o []byte, err error) { + if len(b) < 1 { + err = ErrShortBytes + return + } + + k := NextType(b) + + switch k { + case MapType: + i, o, err = ReadMapStrIntfBytes(b, nil) + return + + case ArrayType: + var sz uint32 + sz, o, err = ReadArrayHeaderBytes(b) + if err != nil { + return + } + j := make([]interface{}, int(sz)) + i = j + for d := range j { + j[d], o, err = ReadIntfBytes(o) + if err != nil { + return + } + } + return + + case Float32Type: + i, o, err = ReadFloat32Bytes(b) + return + + case Float64Type: + i, o, err = ReadFloat64Bytes(b) + return + + case IntType: + i, o, err = ReadInt64Bytes(b) + return + + case UintType: + i, o, err = ReadUint64Bytes(b) + return + + case BoolType: + i, o, err = ReadBoolBytes(b) + return + + case TimeType: + i, o, err = ReadTimeBytes(b) + return + + case Complex64Type: + i, o, err = ReadComplex64Bytes(b) + return + + case Complex128Type: + i, o, err = ReadComplex128Bytes(b) + return + + case ExtensionType: + var t int8 + t, err = peekExtension(b) + if err != nil { + return + } + // use a user-defined extension, + // if it's been registered + f, ok := extensionReg[t] + if ok { + e := f() + o, err = ReadExtensionBytes(b, e) + i = e + return + } + // last resort is a raw extension + e := RawExtension{} + e.Type = int8(t) + o, err = ReadExtensionBytes(b, &e) + i = &e + return + + case NilType: + o, err = ReadNilBytes(b) + return + + case BinType: + i, o, err = ReadBytesBytes(b, nil) + return + + case StrType: + i, o, err = ReadStringBytes(b) + return + + default: + err = InvalidPrefixError(b[0]) + return + } +} + +// Skip skips the next object in 'b' and +// returns the remaining bytes. If the object +// is a map or array, all of its elements +// will be skipped. +// Possible Errors: +// - ErrShortBytes (not enough bytes in b) +// - InvalidPrefixError (bad encoding) +func Skip(b []byte) ([]byte, error) { + sz, asz, err := getSize(b) + if err != nil { + return b, err + } + if uintptr(len(b)) < sz { + return b, ErrShortBytes + } + b = b[sz:] + for asz > 0 { + b, err = Skip(b) + if err != nil { + return b, err + } + asz-- + } + return b, nil +} + +// returns (skip N bytes, skip M objects, error) +func getSize(b []byte) (uintptr, uintptr, error) { + l := len(b) + if l == 0 { + return 0, 0, ErrShortBytes + } + lead := b[0] + spec := &sizes[lead] // get type information + size, mode := spec.size, spec.extra + if size == 0 { + return 0, 0, InvalidPrefixError(lead) + } + if mode >= 0 { // fixed composites + return uintptr(size), uintptr(mode), nil + } + if l < int(size) { + return 0, 0, ErrShortBytes + } + switch mode { + case extra8: + return uintptr(size) + uintptr(b[1]), 0, nil + case extra16: + return uintptr(size) + uintptr(big.Uint16(b[1:])), 0, nil + case extra32: + return uintptr(size) + uintptr(big.Uint32(b[1:])), 0, nil + case map16v: + return uintptr(size), 2 * uintptr(big.Uint16(b[1:])), nil + case map32v: + return uintptr(size), 2 * uintptr(big.Uint32(b[1:])), nil + case array16v: + return uintptr(size), uintptr(big.Uint16(b[1:])), nil + case array32v: + return uintptr(size), uintptr(big.Uint32(b[1:])), nil + default: + return 0, 0, fatal + } +} diff --git a/vendor/github.com/tinylib/msgp/msgp/size.go b/vendor/github.com/tinylib/msgp/msgp/size.go new file mode 100644 index 00000000..ce2f8b16 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/size.go @@ -0,0 +1,38 @@ +package msgp + +// The sizes provided +// are the worst-case +// encoded sizes for +// each type. For variable- +// length types ([]byte, string), +// the total encoded size is +// the prefix size plus the +// length of the object. +const ( + Int64Size = 9 + IntSize = Int64Size + UintSize = Int64Size + Int8Size = 2 + Int16Size = 3 + Int32Size = 5 + Uint8Size = 2 + ByteSize = Uint8Size + Uint16Size = 3 + Uint32Size = 5 + Uint64Size = Int64Size + Float64Size = 9 + Float32Size = 5 + Complex64Size = 10 + Complex128Size = 18 + + TimeSize = 15 + BoolSize = 1 + NilSize = 1 + + MapHeaderSize = 5 + ArrayHeaderSize = 5 + + BytesPrefixSize = 5 + StringPrefixSize = 5 + ExtensionPrefixSize = 6 +) diff --git a/vendor/github.com/tinylib/msgp/msgp/unsafe.go b/vendor/github.com/tinylib/msgp/msgp/unsafe.go new file mode 100644 index 00000000..3978b6ff --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/unsafe.go @@ -0,0 +1,41 @@ +// +build !purego,!appengine + +package msgp + +import ( + "reflect" + "unsafe" +) + +// NOTE: +// all of the definition in this file +// should be repeated in appengine.go, +// but without using unsafe + +const ( + // spec says int and uint are always + // the same size, but that int/uint + // size may not be machine word size + smallint = unsafe.Sizeof(int(0)) == 4 +) + +// UnsafeString returns the byte slice as a volatile string +// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR. +// THIS IS EVIL CODE. +// YOU HAVE BEEN WARNED. +func UnsafeString(b []byte) string { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: sh.Data, Len: sh.Len})) +} + +// UnsafeBytes returns the string as a byte slice +// THIS SHOULD ONLY BE USED BY THE CODE GENERATOR. +// THIS IS EVIL CODE. +// YOU HAVE BEEN WARNED. +func UnsafeBytes(s string) []byte { + return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ + Len: len(s), + Cap: len(s), + Data: (*(*reflect.StringHeader)(unsafe.Pointer(&s))).Data, + })) +} diff --git a/vendor/github.com/tinylib/msgp/msgp/write.go b/vendor/github.com/tinylib/msgp/msgp/write.go new file mode 100644 index 00000000..da9099c2 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/write.go @@ -0,0 +1,845 @@ +package msgp + +import ( + "errors" + "fmt" + "io" + "math" + "reflect" + "sync" + "time" +) + +// Sizer is an interface implemented +// by types that can estimate their +// size when MessagePack encoded. +// This interface is optional, but +// encoding/marshaling implementations +// may use this as a way to pre-allocate +// memory for serialization. +type Sizer interface { + Msgsize() int +} + +var ( + // Nowhere is an io.Writer to nowhere + Nowhere io.Writer = nwhere{} + + btsType = reflect.TypeOf(([]byte)(nil)) + writerPool = sync.Pool{ + New: func() interface{} { + return &Writer{buf: make([]byte, 2048)} + }, + } +) + +func popWriter(w io.Writer) *Writer { + wr := writerPool.Get().(*Writer) + wr.Reset(w) + return wr +} + +func pushWriter(wr *Writer) { + wr.w = nil + wr.wloc = 0 + writerPool.Put(wr) +} + +// freeW frees a writer for use +// by other processes. It is not necessary +// to call freeW on a writer. However, maintaining +// a reference to a *Writer after calling freeW on +// it will cause undefined behavior. +func freeW(w *Writer) { pushWriter(w) } + +// Require ensures that cap(old)-len(old) >= extra. +func Require(old []byte, extra int) []byte { + l := len(old) + c := cap(old) + r := l + extra + if c >= r { + return old + } else if l == 0 { + return make([]byte, 0, extra) + } + // the new size is the greater + // of double the old capacity + // and the sum of the old length + // and the number of new bytes + // necessary. + c <<= 1 + if c < r { + c = r + } + n := make([]byte, l, c) + copy(n, old) + return n +} + +// nowhere writer +type nwhere struct{} + +func (n nwhere) Write(p []byte) (int, error) { return len(p), nil } + +// Marshaler is the interface implemented +// by types that know how to marshal themselves +// as MessagePack. MarshalMsg appends the marshalled +// form of the object to the provided +// byte slice, returning the extended +// slice and any errors encountered. +type Marshaler interface { + MarshalMsg([]byte) ([]byte, error) +} + +// Encodable is the interface implemented +// by types that know how to write themselves +// as MessagePack using a *msgp.Writer. +type Encodable interface { + EncodeMsg(*Writer) error +} + +// Writer is a buffered writer +// that can be used to write +// MessagePack objects to an io.Writer. +// You must call *Writer.Flush() in order +// to flush all of the buffered data +// to the underlying writer. +type Writer struct { + w io.Writer + buf []byte + wloc int +} + +// NewWriter returns a new *Writer. +func NewWriter(w io.Writer) *Writer { + if wr, ok := w.(*Writer); ok { + return wr + } + return popWriter(w) +} + +// NewWriterSize returns a writer with a custom buffer size. +func NewWriterSize(w io.Writer, sz int) *Writer { + // we must be able to require() 18 + // contiguous bytes, so that is the + // practical minimum buffer size + if sz < 18 { + sz = 18 + } + + return &Writer{ + w: w, + buf: make([]byte, sz), + } +} + +// Encode encodes an Encodable to an io.Writer. +func Encode(w io.Writer, e Encodable) error { + wr := NewWriter(w) + err := e.EncodeMsg(wr) + if err == nil { + err = wr.Flush() + } + freeW(wr) + return err +} + +func (mw *Writer) flush() error { + if mw.wloc == 0 { + return nil + } + n, err := mw.w.Write(mw.buf[:mw.wloc]) + if err != nil { + if n > 0 { + mw.wloc = copy(mw.buf, mw.buf[n:mw.wloc]) + } + return err + } + mw.wloc = 0 + return nil +} + +// Flush flushes all of the buffered +// data to the underlying writer. +func (mw *Writer) Flush() error { return mw.flush() } + +// Buffered returns the number bytes in the write buffer +func (mw *Writer) Buffered() int { return len(mw.buf) - mw.wloc } + +func (mw *Writer) avail() int { return len(mw.buf) - mw.wloc } + +func (mw *Writer) bufsize() int { return len(mw.buf) } + +// NOTE: this should only be called with +// a number that is guaranteed to be less than +// len(mw.buf). typically, it is called with a constant. +// +// NOTE: this is a hot code path +func (mw *Writer) require(n int) (int, error) { + c := len(mw.buf) + wl := mw.wloc + if c-wl < n { + if err := mw.flush(); err != nil { + return 0, err + } + wl = mw.wloc + } + mw.wloc += n + return wl, nil +} + +func (mw *Writer) Append(b ...byte) error { + if mw.avail() < len(b) { + err := mw.flush() + if err != nil { + return err + } + } + mw.wloc += copy(mw.buf[mw.wloc:], b) + return nil +} + +// push one byte onto the buffer +// +// NOTE: this is a hot code path +func (mw *Writer) push(b byte) error { + if mw.wloc == len(mw.buf) { + if err := mw.flush(); err != nil { + return err + } + } + mw.buf[mw.wloc] = b + mw.wloc++ + return nil +} + +func (mw *Writer) prefix8(b byte, u uint8) error { + const need = 2 + if len(mw.buf)-mw.wloc < need { + if err := mw.flush(); err != nil { + return err + } + } + prefixu8(mw.buf[mw.wloc:], b, u) + mw.wloc += need + return nil +} + +func (mw *Writer) prefix16(b byte, u uint16) error { + const need = 3 + if len(mw.buf)-mw.wloc < need { + if err := mw.flush(); err != nil { + return err + } + } + prefixu16(mw.buf[mw.wloc:], b, u) + mw.wloc += need + return nil +} + +func (mw *Writer) prefix32(b byte, u uint32) error { + const need = 5 + if len(mw.buf)-mw.wloc < need { + if err := mw.flush(); err != nil { + return err + } + } + prefixu32(mw.buf[mw.wloc:], b, u) + mw.wloc += need + return nil +} + +func (mw *Writer) prefix64(b byte, u uint64) error { + const need = 9 + if len(mw.buf)-mw.wloc < need { + if err := mw.flush(); err != nil { + return err + } + } + prefixu64(mw.buf[mw.wloc:], b, u) + mw.wloc += need + return nil +} + +// Write implements io.Writer, and writes +// data directly to the buffer. +func (mw *Writer) Write(p []byte) (int, error) { + l := len(p) + if mw.avail() < l { + if err := mw.flush(); err != nil { + return 0, err + } + if l > len(mw.buf) { + return mw.w.Write(p) + } + } + mw.wloc += copy(mw.buf[mw.wloc:], p) + return l, nil +} + +// implements io.WriteString +func (mw *Writer) writeString(s string) error { + l := len(s) + if mw.avail() < l { + if err := mw.flush(); err != nil { + return err + } + if l > len(mw.buf) { + _, err := io.WriteString(mw.w, s) + return err + } + } + mw.wloc += copy(mw.buf[mw.wloc:], s) + return nil +} + +// Reset changes the underlying writer used by the Writer +func (mw *Writer) Reset(w io.Writer) { + mw.buf = mw.buf[:cap(mw.buf)] + mw.w = w + mw.wloc = 0 +} + +// WriteMapHeader writes a map header of the given +// size to the writer +func (mw *Writer) WriteMapHeader(sz uint32) error { + switch { + case sz <= 15: + return mw.push(wfixmap(uint8(sz))) + case sz <= math.MaxUint16: + return mw.prefix16(mmap16, uint16(sz)) + default: + return mw.prefix32(mmap32, sz) + } +} + +// WriteArrayHeader writes an array header of the +// given size to the writer +func (mw *Writer) WriteArrayHeader(sz uint32) error { + switch { + case sz <= 15: + return mw.push(wfixarray(uint8(sz))) + case sz <= math.MaxUint16: + return mw.prefix16(marray16, uint16(sz)) + default: + return mw.prefix32(marray32, sz) + } +} + +// WriteNil writes a nil byte to the buffer +func (mw *Writer) WriteNil() error { + return mw.push(mnil) +} + +// WriteFloat64 writes a float64 to the writer +func (mw *Writer) WriteFloat64(f float64) error { + return mw.prefix64(mfloat64, math.Float64bits(f)) +} + +// WriteFloat32 writes a float32 to the writer +func (mw *Writer) WriteFloat32(f float32) error { + return mw.prefix32(mfloat32, math.Float32bits(f)) +} + +// WriteInt64 writes an int64 to the writer +func (mw *Writer) WriteInt64(i int64) error { + if i >= 0 { + switch { + case i <= math.MaxInt8: + return mw.push(wfixint(uint8(i))) + case i <= math.MaxInt16: + return mw.prefix16(mint16, uint16(i)) + case i <= math.MaxInt32: + return mw.prefix32(mint32, uint32(i)) + default: + return mw.prefix64(mint64, uint64(i)) + } + } + switch { + case i >= -32: + return mw.push(wnfixint(int8(i))) + case i >= math.MinInt8: + return mw.prefix8(mint8, uint8(i)) + case i >= math.MinInt16: + return mw.prefix16(mint16, uint16(i)) + case i >= math.MinInt32: + return mw.prefix32(mint32, uint32(i)) + default: + return mw.prefix64(mint64, uint64(i)) + } +} + +// WriteInt8 writes an int8 to the writer +func (mw *Writer) WriteInt8(i int8) error { return mw.WriteInt64(int64(i)) } + +// WriteInt16 writes an int16 to the writer +func (mw *Writer) WriteInt16(i int16) error { return mw.WriteInt64(int64(i)) } + +// WriteInt32 writes an int32 to the writer +func (mw *Writer) WriteInt32(i int32) error { return mw.WriteInt64(int64(i)) } + +// WriteInt writes an int to the writer +func (mw *Writer) WriteInt(i int) error { return mw.WriteInt64(int64(i)) } + +// WriteUint64 writes a uint64 to the writer +func (mw *Writer) WriteUint64(u uint64) error { + switch { + case u <= (1<<7)-1: + return mw.push(wfixint(uint8(u))) + case u <= math.MaxUint8: + return mw.prefix8(muint8, uint8(u)) + case u <= math.MaxUint16: + return mw.prefix16(muint16, uint16(u)) + case u <= math.MaxUint32: + return mw.prefix32(muint32, uint32(u)) + default: + return mw.prefix64(muint64, u) + } +} + +// WriteByte is analogous to WriteUint8 +func (mw *Writer) WriteByte(u byte) error { return mw.WriteUint8(uint8(u)) } + +// WriteUint8 writes a uint8 to the writer +func (mw *Writer) WriteUint8(u uint8) error { return mw.WriteUint64(uint64(u)) } + +// WriteUint16 writes a uint16 to the writer +func (mw *Writer) WriteUint16(u uint16) error { return mw.WriteUint64(uint64(u)) } + +// WriteUint32 writes a uint32 to the writer +func (mw *Writer) WriteUint32(u uint32) error { return mw.WriteUint64(uint64(u)) } + +// WriteUint writes a uint to the writer +func (mw *Writer) WriteUint(u uint) error { return mw.WriteUint64(uint64(u)) } + +// WriteBytes writes binary as 'bin' to the writer +func (mw *Writer) WriteBytes(b []byte) error { + sz := uint32(len(b)) + var err error + switch { + case sz <= math.MaxUint8: + err = mw.prefix8(mbin8, uint8(sz)) + case sz <= math.MaxUint16: + err = mw.prefix16(mbin16, uint16(sz)) + default: + err = mw.prefix32(mbin32, sz) + } + if err != nil { + return err + } + _, err = mw.Write(b) + return err +} + +// WriteBytesHeader writes just the size header +// of a MessagePack 'bin' object. The user is responsible +// for then writing 'sz' more bytes into the stream. +func (mw *Writer) WriteBytesHeader(sz uint32) error { + switch { + case sz <= math.MaxUint8: + return mw.prefix8(mbin8, uint8(sz)) + case sz <= math.MaxUint16: + return mw.prefix16(mbin16, uint16(sz)) + default: + return mw.prefix32(mbin32, sz) + } +} + +// WriteBool writes a bool to the writer +func (mw *Writer) WriteBool(b bool) error { + if b { + return mw.push(mtrue) + } + return mw.push(mfalse) +} + +// WriteString writes a messagepack string to the writer. +// (This is NOT an implementation of io.StringWriter) +func (mw *Writer) WriteString(s string) error { + sz := uint32(len(s)) + var err error + switch { + case sz <= 31: + err = mw.push(wfixstr(uint8(sz))) + case sz <= math.MaxUint8: + err = mw.prefix8(mstr8, uint8(sz)) + case sz <= math.MaxUint16: + err = mw.prefix16(mstr16, uint16(sz)) + default: + err = mw.prefix32(mstr32, sz) + } + if err != nil { + return err + } + return mw.writeString(s) +} + +// WriteStringHeader writes just the string size +// header of a MessagePack 'str' object. The user +// is responsible for writing 'sz' more valid UTF-8 +// bytes to the stream. +func (mw *Writer) WriteStringHeader(sz uint32) error { + switch { + case sz <= 31: + return mw.push(wfixstr(uint8(sz))) + case sz <= math.MaxUint8: + return mw.prefix8(mstr8, uint8(sz)) + case sz <= math.MaxUint16: + return mw.prefix16(mstr16, uint16(sz)) + default: + return mw.prefix32(mstr32, sz) + } +} + +// WriteStringFromBytes writes a 'str' object +// from a []byte. +func (mw *Writer) WriteStringFromBytes(str []byte) error { + sz := uint32(len(str)) + var err error + switch { + case sz <= 31: + err = mw.push(wfixstr(uint8(sz))) + case sz <= math.MaxUint8: + err = mw.prefix8(mstr8, uint8(sz)) + case sz <= math.MaxUint16: + err = mw.prefix16(mstr16, uint16(sz)) + default: + err = mw.prefix32(mstr32, sz) + } + if err != nil { + return err + } + _, err = mw.Write(str) + return err +} + +// WriteComplex64 writes a complex64 to the writer +func (mw *Writer) WriteComplex64(f complex64) error { + o, err := mw.require(10) + if err != nil { + return err + } + mw.buf[o] = mfixext8 + mw.buf[o+1] = Complex64Extension + big.PutUint32(mw.buf[o+2:], math.Float32bits(real(f))) + big.PutUint32(mw.buf[o+6:], math.Float32bits(imag(f))) + return nil +} + +// WriteComplex128 writes a complex128 to the writer +func (mw *Writer) WriteComplex128(f complex128) error { + o, err := mw.require(18) + if err != nil { + return err + } + mw.buf[o] = mfixext16 + mw.buf[o+1] = Complex128Extension + big.PutUint64(mw.buf[o+2:], math.Float64bits(real(f))) + big.PutUint64(mw.buf[o+10:], math.Float64bits(imag(f))) + return nil +} + +// WriteMapStrStr writes a map[string]string to the writer +func (mw *Writer) WriteMapStrStr(mp map[string]string) (err error) { + err = mw.WriteMapHeader(uint32(len(mp))) + if err != nil { + return + } + for key, val := range mp { + err = mw.WriteString(key) + if err != nil { + return + } + err = mw.WriteString(val) + if err != nil { + return + } + } + return nil +} + +// WriteMapStrIntf writes a map[string]interface to the writer +func (mw *Writer) WriteMapStrIntf(mp map[string]interface{}) (err error) { + err = mw.WriteMapHeader(uint32(len(mp))) + if err != nil { + return + } + for key, val := range mp { + err = mw.WriteString(key) + if err != nil { + return + } + err = mw.WriteIntf(val) + if err != nil { + return + } + } + return +} + +// WriteTime writes a time.Time object to the wire. +// +// Time is encoded as Unix time, which means that +// location (time zone) data is removed from the object. +// The encoded object itself is 12 bytes: 8 bytes for +// a big-endian 64-bit integer denoting seconds +// elapsed since "zero" Unix time, followed by 4 bytes +// for a big-endian 32-bit signed integer denoting +// the nanosecond offset of the time. This encoding +// is intended to ease portability across languages. +// (Note that this is *not* the standard time.Time +// binary encoding, because its implementation relies +// heavily on the internal representation used by the +// time package.) +func (mw *Writer) WriteTime(t time.Time) error { + t = t.UTC() + o, err := mw.require(15) + if err != nil { + return err + } + mw.buf[o] = mext8 + mw.buf[o+1] = 12 + mw.buf[o+2] = TimeExtension + putUnix(mw.buf[o+3:], t.Unix(), int32(t.Nanosecond())) + return nil +} + +// WriteIntf writes the concrete type of 'v'. +// WriteIntf will error if 'v' is not one of the following: +// - A bool, float, string, []byte, int, uint, or complex +// - A map of supported types (with string keys) +// - An array or slice of supported types +// - A pointer to a supported type +// - A type that satisfies the msgp.Encodable interface +// - A type that satisfies the msgp.Extension interface +func (mw *Writer) WriteIntf(v interface{}) error { + if v == nil { + return mw.WriteNil() + } + switch v := v.(type) { + + // preferred interfaces + + case Encodable: + return v.EncodeMsg(mw) + case Extension: + return mw.WriteExtension(v) + + // concrete types + + case bool: + return mw.WriteBool(v) + case float32: + return mw.WriteFloat32(v) + case float64: + return mw.WriteFloat64(v) + case complex64: + return mw.WriteComplex64(v) + case complex128: + return mw.WriteComplex128(v) + case uint8: + return mw.WriteUint8(v) + case uint16: + return mw.WriteUint16(v) + case uint32: + return mw.WriteUint32(v) + case uint64: + return mw.WriteUint64(v) + case uint: + return mw.WriteUint(v) + case int8: + return mw.WriteInt8(v) + case int16: + return mw.WriteInt16(v) + case int32: + return mw.WriteInt32(v) + case int64: + return mw.WriteInt64(v) + case int: + return mw.WriteInt(v) + case string: + return mw.WriteString(v) + case []byte: + return mw.WriteBytes(v) + case map[string]string: + return mw.WriteMapStrStr(v) + case map[string]interface{}: + return mw.WriteMapStrIntf(v) + case time.Time: + return mw.WriteTime(v) + } + + val := reflect.ValueOf(v) + if !isSupported(val.Kind()) || !val.IsValid() { + return fmt.Errorf("msgp: type %s not supported", val) + } + + switch val.Kind() { + case reflect.Ptr: + if val.IsNil() { + return mw.WriteNil() + } + return mw.WriteIntf(val.Elem().Interface()) + case reflect.Slice: + return mw.writeSlice(val) + case reflect.Map: + return mw.writeMap(val) + } + return &ErrUnsupportedType{val.Type()} +} + +func (mw *Writer) writeMap(v reflect.Value) (err error) { + if v.Type().Key().Kind() != reflect.String { + return errors.New("msgp: map keys must be strings") + } + ks := v.MapKeys() + err = mw.WriteMapHeader(uint32(len(ks))) + if err != nil { + return + } + for _, key := range ks { + val := v.MapIndex(key) + err = mw.WriteString(key.String()) + if err != nil { + return + } + err = mw.WriteIntf(val.Interface()) + if err != nil { + return + } + } + return +} + +func (mw *Writer) writeSlice(v reflect.Value) (err error) { + // is []byte + if v.Type().ConvertibleTo(btsType) { + return mw.WriteBytes(v.Bytes()) + } + + sz := uint32(v.Len()) + err = mw.WriteArrayHeader(sz) + if err != nil { + return + } + for i := uint32(0); i < sz; i++ { + err = mw.WriteIntf(v.Index(int(i)).Interface()) + if err != nil { + return + } + } + return +} + +func (mw *Writer) writeStruct(v reflect.Value) error { + if enc, ok := v.Interface().(Encodable); ok { + return enc.EncodeMsg(mw) + } + return fmt.Errorf("msgp: unsupported type: %s", v.Type()) +} + +func (mw *Writer) writeVal(v reflect.Value) error { + if !isSupported(v.Kind()) { + return fmt.Errorf("msgp: msgp/enc: type %q not supported", v.Type()) + } + + // shortcut for nil values + if v.IsNil() { + return mw.WriteNil() + } + switch v.Kind() { + case reflect.Bool: + return mw.WriteBool(v.Bool()) + + case reflect.Float32, reflect.Float64: + return mw.WriteFloat64(v.Float()) + + case reflect.Complex64, reflect.Complex128: + return mw.WriteComplex128(v.Complex()) + + case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: + return mw.WriteInt64(v.Int()) + + case reflect.Interface, reflect.Ptr: + if v.IsNil() { + mw.WriteNil() + } + return mw.writeVal(v.Elem()) + + case reflect.Map: + return mw.writeMap(v) + + case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint8: + return mw.WriteUint64(v.Uint()) + + case reflect.String: + return mw.WriteString(v.String()) + + case reflect.Slice, reflect.Array: + return mw.writeSlice(v) + + case reflect.Struct: + return mw.writeStruct(v) + + } + return fmt.Errorf("msgp: msgp/enc: type %q not supported", v.Type()) +} + +// is the reflect.Kind encodable? +func isSupported(k reflect.Kind) bool { + switch k { + case reflect.Func, reflect.Chan, reflect.Invalid, reflect.UnsafePointer: + return false + default: + return true + } +} + +// GuessSize guesses the size of the underlying +// value of 'i'. If the underlying value is not +// a simple builtin (or []byte), GuessSize defaults +// to 512. +func GuessSize(i interface{}) int { + if i == nil { + return NilSize + } + + switch i := i.(type) { + case Sizer: + return i.Msgsize() + case Extension: + return ExtensionPrefixSize + i.Len() + case float64: + return Float64Size + case float32: + return Float32Size + case uint8, uint16, uint32, uint64, uint: + return UintSize + case int8, int16, int32, int64, int: + return IntSize + case []byte: + return BytesPrefixSize + len(i) + case string: + return StringPrefixSize + len(i) + case complex64: + return Complex64Size + case complex128: + return Complex128Size + case bool: + return BoolSize + case map[string]interface{}: + s := MapHeaderSize + for key, val := range i { + s += StringPrefixSize + len(key) + GuessSize(val) + } + return s + case map[string]string: + s := MapHeaderSize + for key, val := range i { + s += 2*StringPrefixSize + len(key) + len(val) + } + return s + default: + return 512 + } +} diff --git a/vendor/github.com/tinylib/msgp/msgp/write_bytes.go b/vendor/github.com/tinylib/msgp/msgp/write_bytes.go new file mode 100644 index 00000000..eaa03c46 --- /dev/null +++ b/vendor/github.com/tinylib/msgp/msgp/write_bytes.go @@ -0,0 +1,411 @@ +package msgp + +import ( + "math" + "reflect" + "time" +) + +// ensure 'sz' extra bytes in 'b' btw len(b) and cap(b) +func ensure(b []byte, sz int) ([]byte, int) { + l := len(b) + c := cap(b) + if c-l < sz { + o := make([]byte, (2*c)+sz) // exponential growth + n := copy(o, b) + return o[:n+sz], n + } + return b[:l+sz], l +} + +// AppendMapHeader appends a map header with the +// given size to the slice +func AppendMapHeader(b []byte, sz uint32) []byte { + switch { + case sz <= 15: + return append(b, wfixmap(uint8(sz))) + + case sz <= math.MaxUint16: + o, n := ensure(b, 3) + prefixu16(o[n:], mmap16, uint16(sz)) + return o + + default: + o, n := ensure(b, 5) + prefixu32(o[n:], mmap32, sz) + return o + } +} + +// AppendArrayHeader appends an array header with +// the given size to the slice +func AppendArrayHeader(b []byte, sz uint32) []byte { + switch { + case sz <= 15: + return append(b, wfixarray(uint8(sz))) + + case sz <= math.MaxUint16: + o, n := ensure(b, 3) + prefixu16(o[n:], marray16, uint16(sz)) + return o + + default: + o, n := ensure(b, 5) + prefixu32(o[n:], marray32, sz) + return o + } +} + +// AppendNil appends a 'nil' byte to the slice +func AppendNil(b []byte) []byte { return append(b, mnil) } + +// AppendFloat64 appends a float64 to the slice +func AppendFloat64(b []byte, f float64) []byte { + o, n := ensure(b, Float64Size) + prefixu64(o[n:], mfloat64, math.Float64bits(f)) + return o +} + +// AppendFloat32 appends a float32 to the slice +func AppendFloat32(b []byte, f float32) []byte { + o, n := ensure(b, Float32Size) + prefixu32(o[n:], mfloat32, math.Float32bits(f)) + return o +} + +// AppendInt64 appends an int64 to the slice +func AppendInt64(b []byte, i int64) []byte { + if i >= 0 { + switch { + case i <= math.MaxInt8: + return append(b, wfixint(uint8(i))) + case i <= math.MaxInt16: + o, n := ensure(b, 3) + putMint16(o[n:], int16(i)) + return o + case i <= math.MaxInt32: + o, n := ensure(b, 5) + putMint32(o[n:], int32(i)) + return o + default: + o, n := ensure(b, 9) + putMint64(o[n:], i) + return o + } + } + switch { + case i >= -32: + return append(b, wnfixint(int8(i))) + case i >= math.MinInt8: + o, n := ensure(b, 2) + putMint8(o[n:], int8(i)) + return o + case i >= math.MinInt16: + o, n := ensure(b, 3) + putMint16(o[n:], int16(i)) + return o + case i >= math.MinInt32: + o, n := ensure(b, 5) + putMint32(o[n:], int32(i)) + return o + default: + o, n := ensure(b, 9) + putMint64(o[n:], i) + return o + } +} + +// AppendInt appends an int to the slice +func AppendInt(b []byte, i int) []byte { return AppendInt64(b, int64(i)) } + +// AppendInt8 appends an int8 to the slice +func AppendInt8(b []byte, i int8) []byte { return AppendInt64(b, int64(i)) } + +// AppendInt16 appends an int16 to the slice +func AppendInt16(b []byte, i int16) []byte { return AppendInt64(b, int64(i)) } + +// AppendInt32 appends an int32 to the slice +func AppendInt32(b []byte, i int32) []byte { return AppendInt64(b, int64(i)) } + +// AppendUint64 appends a uint64 to the slice +func AppendUint64(b []byte, u uint64) []byte { + switch { + case u <= (1<<7)-1: + return append(b, wfixint(uint8(u))) + + case u <= math.MaxUint8: + o, n := ensure(b, 2) + putMuint8(o[n:], uint8(u)) + return o + + case u <= math.MaxUint16: + o, n := ensure(b, 3) + putMuint16(o[n:], uint16(u)) + return o + + case u <= math.MaxUint32: + o, n := ensure(b, 5) + putMuint32(o[n:], uint32(u)) + return o + + default: + o, n := ensure(b, 9) + putMuint64(o[n:], u) + return o + + } +} + +// AppendUint appends a uint to the slice +func AppendUint(b []byte, u uint) []byte { return AppendUint64(b, uint64(u)) } + +// AppendUint8 appends a uint8 to the slice +func AppendUint8(b []byte, u uint8) []byte { return AppendUint64(b, uint64(u)) } + +// AppendByte is analogous to AppendUint8 +func AppendByte(b []byte, u byte) []byte { return AppendUint8(b, uint8(u)) } + +// AppendUint16 appends a uint16 to the slice +func AppendUint16(b []byte, u uint16) []byte { return AppendUint64(b, uint64(u)) } + +// AppendUint32 appends a uint32 to the slice +func AppendUint32(b []byte, u uint32) []byte { return AppendUint64(b, uint64(u)) } + +// AppendBytes appends bytes to the slice as MessagePack 'bin' data +func AppendBytes(b []byte, bts []byte) []byte { + sz := len(bts) + var o []byte + var n int + switch { + case sz <= math.MaxUint8: + o, n = ensure(b, 2+sz) + prefixu8(o[n:], mbin8, uint8(sz)) + n += 2 + case sz <= math.MaxUint16: + o, n = ensure(b, 3+sz) + prefixu16(o[n:], mbin16, uint16(sz)) + n += 3 + default: + o, n = ensure(b, 5+sz) + prefixu32(o[n:], mbin32, uint32(sz)) + n += 5 + } + return o[:n+copy(o[n:], bts)] +} + +// AppendBool appends a bool to the slice +func AppendBool(b []byte, t bool) []byte { + if t { + return append(b, mtrue) + } + return append(b, mfalse) +} + +// AppendString appends a string as a MessagePack 'str' to the slice +func AppendString(b []byte, s string) []byte { + sz := len(s) + var n int + var o []byte + switch { + case sz <= 31: + o, n = ensure(b, 1+sz) + o[n] = wfixstr(uint8(sz)) + n++ + case sz <= math.MaxUint8: + o, n = ensure(b, 2+sz) + prefixu8(o[n:], mstr8, uint8(sz)) + n += 2 + case sz <= math.MaxUint16: + o, n = ensure(b, 3+sz) + prefixu16(o[n:], mstr16, uint16(sz)) + n += 3 + default: + o, n = ensure(b, 5+sz) + prefixu32(o[n:], mstr32, uint32(sz)) + n += 5 + } + return o[:n+copy(o[n:], s)] +} + +// AppendStringFromBytes appends a []byte +// as a MessagePack 'str' to the slice 'b.' +func AppendStringFromBytes(b []byte, str []byte) []byte { + sz := len(str) + var n int + var o []byte + switch { + case sz <= 31: + o, n = ensure(b, 1+sz) + o[n] = wfixstr(uint8(sz)) + n++ + case sz <= math.MaxUint8: + o, n = ensure(b, 2+sz) + prefixu8(o[n:], mstr8, uint8(sz)) + n += 2 + case sz <= math.MaxUint16: + o, n = ensure(b, 3+sz) + prefixu16(o[n:], mstr16, uint16(sz)) + n += 3 + default: + o, n = ensure(b, 5+sz) + prefixu32(o[n:], mstr32, uint32(sz)) + n += 5 + } + return o[:n+copy(o[n:], str)] +} + +// AppendComplex64 appends a complex64 to the slice as a MessagePack extension +func AppendComplex64(b []byte, c complex64) []byte { + o, n := ensure(b, Complex64Size) + o[n] = mfixext8 + o[n+1] = Complex64Extension + big.PutUint32(o[n+2:], math.Float32bits(real(c))) + big.PutUint32(o[n+6:], math.Float32bits(imag(c))) + return o +} + +// AppendComplex128 appends a complex128 to the slice as a MessagePack extension +func AppendComplex128(b []byte, c complex128) []byte { + o, n := ensure(b, Complex128Size) + o[n] = mfixext16 + o[n+1] = Complex128Extension + big.PutUint64(o[n+2:], math.Float64bits(real(c))) + big.PutUint64(o[n+10:], math.Float64bits(imag(c))) + return o +} + +// AppendTime appends a time.Time to the slice as a MessagePack extension +func AppendTime(b []byte, t time.Time) []byte { + o, n := ensure(b, TimeSize) + t = t.UTC() + o[n] = mext8 + o[n+1] = 12 + o[n+2] = TimeExtension + putUnix(o[n+3:], t.Unix(), int32(t.Nanosecond())) + return o +} + +// AppendMapStrStr appends a map[string]string to the slice +// as a MessagePack map with 'str'-type keys and values +func AppendMapStrStr(b []byte, m map[string]string) []byte { + sz := uint32(len(m)) + b = AppendMapHeader(b, sz) + for key, val := range m { + b = AppendString(b, key) + b = AppendString(b, val) + } + return b +} + +// AppendMapStrIntf appends a map[string]interface{} to the slice +// as a MessagePack map with 'str'-type keys. +func AppendMapStrIntf(b []byte, m map[string]interface{}) ([]byte, error) { + sz := uint32(len(m)) + b = AppendMapHeader(b, sz) + var err error + for key, val := range m { + b = AppendString(b, key) + b, err = AppendIntf(b, val) + if err != nil { + return b, err + } + } + return b, nil +} + +// AppendIntf appends the concrete type of 'i' to the +// provided []byte. 'i' must be one of the following: +// - 'nil' +// - A bool, float, string, []byte, int, uint, or complex +// - A map[string]interface{} or map[string]string +// - A []T, where T is another supported type +// - A *T, where T is another supported type +// - A type that satisfieds the msgp.Marshaler interface +// - A type that satisfies the msgp.Extension interface +func AppendIntf(b []byte, i interface{}) ([]byte, error) { + if i == nil { + return AppendNil(b), nil + } + + // all the concrete types + // for which we have methods + switch i := i.(type) { + case Marshaler: + return i.MarshalMsg(b) + case Extension: + return AppendExtension(b, i) + case bool: + return AppendBool(b, i), nil + case float32: + return AppendFloat32(b, i), nil + case float64: + return AppendFloat64(b, i), nil + case complex64: + return AppendComplex64(b, i), nil + case complex128: + return AppendComplex128(b, i), nil + case string: + return AppendString(b, i), nil + case []byte: + return AppendBytes(b, i), nil + case int8: + return AppendInt8(b, i), nil + case int16: + return AppendInt16(b, i), nil + case int32: + return AppendInt32(b, i), nil + case int64: + return AppendInt64(b, i), nil + case int: + return AppendInt64(b, int64(i)), nil + case uint: + return AppendUint64(b, uint64(i)), nil + case uint8: + return AppendUint8(b, i), nil + case uint16: + return AppendUint16(b, i), nil + case uint32: + return AppendUint32(b, i), nil + case uint64: + return AppendUint64(b, i), nil + case time.Time: + return AppendTime(b, i), nil + case map[string]interface{}: + return AppendMapStrIntf(b, i) + case map[string]string: + return AppendMapStrStr(b, i), nil + case []interface{}: + b = AppendArrayHeader(b, uint32(len(i))) + var err error + for _, k := range i { + b, err = AppendIntf(b, k) + if err != nil { + return b, err + } + } + return b, nil + } + + var err error + v := reflect.ValueOf(i) + switch v.Kind() { + case reflect.Array, reflect.Slice: + l := v.Len() + b = AppendArrayHeader(b, uint32(l)) + for i := 0; i < l; i++ { + b, err = AppendIntf(b, v.Index(i).Interface()) + if err != nil { + return b, err + } + } + return b, nil + case reflect.Ptr: + if v.IsNil() { + return AppendNil(b), err + } + b, err = AppendIntf(b, v.Elem().Interface()) + return b, err + default: + return b, &ErrUnsupportedType{T: v.Type()} + } +} diff --git a/vendor/github.com/willf/bitset/LICENSE b/vendor/github.com/willf/bitset/LICENSE new file mode 100644 index 00000000..59cab8a9 --- /dev/null +++ b/vendor/github.com/willf/bitset/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 Will Fitzgerald. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/willf/bitset/bitset.go b/vendor/github.com/willf/bitset/bitset.go new file mode 100644 index 00000000..9c7d1c6b --- /dev/null +++ b/vendor/github.com/willf/bitset/bitset.go @@ -0,0 +1,719 @@ +/* +Package bitset implements bitsets, a mapping +between non-negative integers and boolean values. It should be more +efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing +individual integers. + +But it also provides set intersection, union, difference, +complement, and symmetric operations, as well as tests to +check whether any, all, or no bits are set, and querying a +bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the +memory allocation is approximately Max bits, where Max is +the largest set bit. BitSets are never shrunk. On creation, +a hint can be given for the number of bits that will be used. + +Many of the methods, including Set,Clear, and Flip, return +a BitSet pointer, which allows for chaining. + +Example use: + + import "bitset" + var b BitSet + b.Set(10).Set(11) + if b.Test(1000) { + b.Clear(1000) + } + if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { + fmt.Println("Intersection works.") + } + +As an alternative to BitSets, one should check out the 'big' package, +which provides a (less set-theoretical) view of bitsets. + +*/ +package bitset + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" +) + +// the wordSize of a bit set +const wordSize = uint(64) + +// log2WordSize is lg(wordSize) +const log2WordSize = uint(6) + +// allBits has every bit set +const allBits uint64 = 0xffffffffffffffff + +// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. +type BitSet struct { + length uint + set []uint64 +} + +// Error is used to distinguish errors (panics) generated in this package. +type Error string + +// safeSet will fixup b.set to be non-nil and return the field value +func (b *BitSet) safeSet() []uint64 { + if b.set == nil { + b.set = make([]uint64, wordsNeeded(0)) + } + return b.set +} + +// From is a constructor used to create a BitSet from an array of integers +func From(buf []uint64) *BitSet { + return &BitSet{uint(len(buf)) * 64, buf} +} + +// Bytes returns the bitset as array of integers +func (b *BitSet) Bytes() []uint64 { + return b.set +} + +// wordsNeeded calculates the number of words needed for i bits +func wordsNeeded(i uint) int { + if i > ((^uint(0)) - wordSize + 1) { + return int((^uint(0)) >> log2WordSize) + } + return int((i + (wordSize - 1)) >> log2WordSize) +} + +// New creates a new BitSet with a hint that length bits will be required +func New(length uint) (bset *BitSet) { + defer func() { + if r := recover(); r != nil { + bset = &BitSet{ + 0, + make([]uint64, 0), + } + } + }() + + bset = &BitSet{ + length, + make([]uint64, wordsNeeded(length)), + } + + return bset +} + +// Cap returns the total possible capicity, or number of bits +func Cap() uint { + return ^uint(0) +} + +// Len returns the length of the BitSet in words +func (b *BitSet) Len() uint { + return b.length +} + +// extendSetMaybe adds additional words to incorporate new bits if needed +func (b *BitSet) extendSetMaybe(i uint) { + if i >= b.length { // if we need more bits, make 'em + nsize := wordsNeeded(i + 1) + if b.set == nil { + b.set = make([]uint64, nsize) + } else if cap(b.set) >= nsize { + b.set = b.set[:nsize] // fast resize + } else if len(b.set) < nsize { + newset := make([]uint64, nsize, 2*nsize) // increase capacity 2x + copy(newset, b.set) + b.set = newset + } + b.length = i + 1 + } +} + +// Test whether bit i is set. +func (b *BitSet) Test(i uint) bool { + if i >= b.length { + return false + } + return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0 +} + +// Set bit i to 1 +func (b *BitSet) Set(i uint) *BitSet { + b.extendSetMaybe(i) + b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1)) + return b +} + +// Clear bit i to 0 +func (b *BitSet) Clear(i uint) *BitSet { + if i >= b.length { + return b + } + b.set[i>>log2WordSize] &^= 1 << (i & (wordSize - 1)) + return b +} + +// SetTo sets bit i to value +func (b *BitSet) SetTo(i uint, value bool) *BitSet { + if value { + return b.Set(i) + } + return b.Clear(i) +} + +// Flip bit at i +func (b *BitSet) Flip(i uint) *BitSet { + if i >= b.length { + return b.Set(i) + } + b.set[i>>log2WordSize] ^= 1 << (i & (wordSize - 1)) + return b +} + +// String creates a string representation of the Bitmap +func (b *BitSet) String() string { + // follows code from https://github.com/RoaringBitmap/roaring + var buffer bytes.Buffer + start := []byte("{") + buffer.Write(start) + counter := 0 + i, e := b.NextSet(0) + for e { + counter = counter + 1 + // to avoid exhausting the memory + if counter > 0x40000 { + buffer.WriteString("...") + break + } + buffer.WriteString(strconv.FormatInt(int64(i), 10)) + i, e = b.NextSet(i + 1) + if e { + buffer.WriteString(",") + } + } + buffer.WriteString("}") + return buffer.String() +} + +// NextSet returns the next bit set from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no set bit found) +// for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...} +func (b *BitSet) NextSet(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> (i & (wordSize - 1)) + if w != 0 { + return i + trailingZeroes64(w), true + } + x = x + 1 + for x < len(b.set) { + if b.set[x] != 0 { + return uint(x)*wordSize + trailingZeroes64(b.set[x]), true + } + x = x + 1 + + } + return 0, false +} + +// NextClear returns the next clear bit from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no bit found i.e. all bits are set) +func (b *BitSet) NextClear(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> (i & (wordSize - 1)) + wA := allBits >> (i & (wordSize - 1)) + if w != wA { + return i + trailingZeroes64(^w), true + } + x++ + for x < len(b.set) { + if b.set[x] != allBits { + return uint(x)*wordSize + trailingZeroes64(^b.set[x]), true + } + x++ + } + return 0, false +} + +// ClearAll clears the entire BitSet +func (b *BitSet) ClearAll() *BitSet { + if b != nil && b.set != nil { + for i := range b.set { + b.set[i] = 0 + } + } + return b +} + +// wordCount returns the number of words used in a bit set +func (b *BitSet) wordCount() int { + return len(b.set) +} + +// Clone this BitSet +func (b *BitSet) Clone() *BitSet { + c := New(b.length) + if b.set != nil { // Clone should not modify current object + copy(c.set, b.set) + } + return c +} + +// Copy into a destination BitSet +// Returning the size of the destination BitSet +// like array copy +func (b *BitSet) Copy(c *BitSet) (count uint) { + if c == nil { + return + } + if b.set != nil { // Copy should not modify current object + copy(c.set, b.set) + } + count = c.length + if b.length < c.length { + count = b.length + } + return +} + +// Count (number of set bits) +func (b *BitSet) Count() uint { + if b != nil && b.set != nil { + return uint(popcntSlice(b.set)) + } + return 0 +} + +var deBruijn = [...]byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +func trailingZeroes64(v uint64) uint { + return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) +} + +// Equal tests the equvalence of two BitSets. +// False if they are of different sizes, otherwise true +// only if all the same bits are set +func (b *BitSet) Equal(c *BitSet) bool { + if c == nil { + return false + } + if b.length != c.length { + return false + } + if b.length == 0 { // if they have both length == 0, then could have nil set + return true + } + // testing for equality shoud not transform the bitset (no call to safeSet) + + for p, v := range b.set { + if c.set[p] != v { + return false + } + } + return true +} + +func panicIfNull(b *BitSet) { + if b == nil { + panic(Error("BitSet must not be null")) + } +} + +// Difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) Difference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + result = b.Clone() // clone b (in case b is bigger than compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + result.set[i] = b.set[i] &^ compare.set[i] + } + return +} + +// DifferenceCardinality computes the cardinality of the differnce +func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + cnt := uint64(0) + cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + cnt += popcntSlice(b.set[l:]) + return uint(cnt) +} + +// InPlaceDifference computes the difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) InPlaceDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &^= compare.set[i] + } +} + +// Convenience function: return two bitsets ordered by +// increasing length. Note: neither can be nil +func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { + if a.length <= b.length { + ap, bp = a, b + } else { + ap, bp = b, a + } + return +} + +// Intersection of base set and other set +// This is the BitSet equivalent of & (and) +func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = New(b.length) + for i, word := range b.set { + result.set[i] = word & compare.set[i] + } + return +} + +// IntersectionCardinality computes the cardinality of the union +func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntAndSlice(b.set, compare.set) + return uint(cnt) +} + +// InPlaceIntersection destructively computes the intersection of +// base set and the compare set. +// This is the BitSet equivalent of & (and) +func (b *BitSet) InPlaceIntersection(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &= compare.set[i] + } + for i := l; i < len(b.set); i++ { + b.set[i] = 0 + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + return +} + +// Union of base set and other set +// This is the BitSet equivalent of | (or) +func (b *BitSet) Union(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word | compare.set[i] + } + return +} + +// UnionCardinality computes the cardinality of the uniton of the base set +// and the compare set. +func (b *BitSet) UnionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntOrSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceUnion creates the destructive union of base set and compare set. +// This is the BitSet equivalent of | (or). +func (b *BitSet) InPlaceUnion(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] |= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) SymmetricDifference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + // compare is bigger, so clone it + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word ^ compare.set[i] + } + return +} + +// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference +func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntXorSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] ^= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// Is the length an exact multiple of word sizes? +func (b *BitSet) isLenExactMultiple() bool { + return b.length%wordSize == 0 +} + +// Clean last word by setting unused bits to 0 +func (b *BitSet) cleanLastWord() { + if !b.isLenExactMultiple() { + b.set[len(b.set)-1] &= allBits >> (wordSize - b.length%wordSize) + } +} + +// Complement computes the (local) complement of a biset (up to length bits) +func (b *BitSet) Complement() (result *BitSet) { + panicIfNull(b) + result = New(b.length) + for i, word := range b.set { + result.set[i] = ^word + } + result.cleanLastWord() + return +} + +// All returns true if all bits are set, false otherwise. Returns true for +// empty sets. +func (b *BitSet) All() bool { + panicIfNull(b) + return b.Count() == b.length +} + +// None returns true if no bit is set, false otherwise. Retursn true for +// empty sets. +func (b *BitSet) None() bool { + panicIfNull(b) + if b != nil && b.set != nil { + for _, word := range b.set { + if word > 0 { + return false + } + } + return true + } + return true +} + +// Any returns true if any bit is set, false otherwise +func (b *BitSet) Any() bool { + panicIfNull(b) + return !b.None() +} + +// IsSuperSet returns true if this is a superset of the other set +func (b *BitSet) IsSuperSet(other *BitSet) bool { + for i, e := other.NextSet(0); e; i, e = other.NextSet(i + 1) { + if !b.Test(i) { + return false + } + } + return true +} + +// IsStrictSuperSet returns true if this is a strict superset of the other set +func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { + return b.Count() > other.Count() && b.IsSuperSet(other) +} + +// DumpAsBits dumps a bit set as a string of bits +func (b *BitSet) DumpAsBits() string { + if b.set == nil { + return "." + } + buffer := bytes.NewBufferString("") + i := len(b.set) - 1 + for ; i >= 0; i-- { + fmt.Fprintf(buffer, "%064b.", b.set[i]) + } + return string(buffer.Bytes()) +} + +// BinaryStorageSize returns the binary storage requirements +func (b *BitSet) BinaryStorageSize() int { + return binary.Size(uint64(0)) + binary.Size(b.set) +} + +// WriteTo writes a BitSet to a stream +func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { + length := uint64(b.length) + + // Write length + err := binary.Write(stream, binary.BigEndian, length) + if err != nil { + return 0, err + } + + // Write set + err = binary.Write(stream, binary.BigEndian, b.set) + return int64(b.BinaryStorageSize()), err +} + +// ReadFrom reads a BitSet from a stream written using WriteTo +func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { + var length uint64 + + // Read length first + err := binary.Read(stream, binary.BigEndian, &length) + if err != nil { + return 0, err + } + newset := New(uint(length)) + + if uint64(newset.length) != length { + return 0, errors.New("Unmarshalling error: type mismatch") + } + + // Read remaining bytes as set + err = binary.Read(stream, binary.BigEndian, newset.set) + if err != nil { + return 0, err + } + + *b = *newset + return int64(b.BinaryStorageSize()), nil +} + +// MarshalBinary encodes a BitSet into a binary form and returns the result. +func (b *BitSet) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + writer := bufio.NewWriter(&buf) + + _, err := b.WriteTo(writer) + if err != nil { + return []byte{}, err + } + + err = writer.Flush() + + return buf.Bytes(), err +} + +// UnmarshalBinary decodes the binary form generated by MarshalBinary. +func (b *BitSet) UnmarshalBinary(data []byte) error { + buf := bytes.NewReader(data) + reader := bufio.NewReader(buf) + + _, err := b.ReadFrom(reader) + + return err +} + +// MarshalJSON marshals a BitSet as a JSON structure +func (b *BitSet) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize())) + _, err := b.WriteTo(buffer) + if err != nil { + return nil, err + } + + // URLEncode all bytes + return json.Marshal(base64.URLEncoding.EncodeToString(buffer.Bytes())) +} + +// UnmarshalJSON unmarshals a BitSet from JSON created using MarshalJSON +func (b *BitSet) UnmarshalJSON(data []byte) error { + // Unmarshal as string + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + // URLDecode string + buf, err := base64.URLEncoding.DecodeString(s) + if err != nil { + return err + } + + _, err = b.ReadFrom(bytes.NewReader(buf)) + return err +} diff --git a/vendor/github.com/willf/bitset/popcnt.go b/vendor/github.com/willf/bitset/popcnt.go new file mode 100644 index 00000000..76577a83 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt.go @@ -0,0 +1,53 @@ +package bitset + +// bit population count, take from +// https://code.google.com/p/go/issues/detail?id=4988#c11 +// credit: https://code.google.com/u/arnehormann/ +func popcount(x uint64) (n uint64) { + x -= (x >> 1) & 0x5555555555555555 + x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 + x += x >> 4 + x &= 0x0f0f0f0f0f0f0f0f + x *= 0x0101010101010101 + return x >> 56 +} + +func popcntSliceGo(s []uint64) uint64 { + cnt := uint64(0) + for _, x := range s { + cnt += popcount(x) + } + return cnt +} + +func popcntMaskSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] &^ m[i]) + } + return cnt +} + +func popcntAndSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] & m[i]) + } + return cnt +} + +func popcntOrSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] | m[i]) + } + return cnt +} + +func popcntXorSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] ^ m[i]) + } + return cnt +} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.go b/vendor/github.com/willf/bitset/popcnt_amd64.go new file mode 100644 index 00000000..665a8644 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64.go @@ -0,0 +1,67 @@ +// +build amd64,!appengine + +package bitset + +// *** the following functions are defined in popcnt_amd64.s + +//go:noescape + +func hasAsm() bool + +// useAsm is a flag used to select the GO or ASM implementation of the popcnt function +var useAsm = hasAsm() + +//go:noescape + +func popcntSliceAsm(s []uint64) uint64 + +//go:noescape + +func popcntMaskSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntAndSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntOrSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntXorSliceAsm(s, m []uint64) uint64 + +func popcntSlice(s []uint64) uint64 { + if useAsm { + return popcntSliceAsm(s) + } + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + if useAsm { + return popcntMaskSliceAsm(s, m) + } + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + if useAsm { + return popcntAndSliceAsm(s, m) + } + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + if useAsm { + return popcntOrSliceAsm(s, m) + } + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + if useAsm { + return popcntXorSliceAsm(s, m) + } + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.s b/vendor/github.com/willf/bitset/popcnt_amd64.s new file mode 100644 index 00000000..18f58784 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64.s @@ -0,0 +1,103 @@ +// +build amd64,!appengine + +TEXT ·hasAsm(SB),4,$0-1 +MOVQ $1, AX +CPUID +SHRQ $23, CX +ANDQ $1, CX +MOVB CX, ret+0(FP) +RET + +#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 + +TEXT ·popcntSliceAsm(SB),4,$0-32 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntSliceEnd +popcntSliceLoop: +BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX +ADDQ DX, AX +ADDQ $8, SI +LOOP popcntSliceLoop +popcntSliceEnd: +MOVQ AX, ret+24(FP) +RET + +TEXT ·popcntMaskSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntMaskSliceEnd +MOVQ m+24(FP), DI +popcntMaskSliceLoop: +MOVQ (DI), DX +NOTQ DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntMaskSliceLoop +popcntMaskSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntAndSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntAndSliceEnd +MOVQ m+24(FP), DI +popcntAndSliceLoop: +MOVQ (DI), DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntAndSliceLoop +popcntAndSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntOrSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntOrSliceEnd +MOVQ m+24(FP), DI +popcntOrSliceLoop: +MOVQ (DI), DX +ORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntOrSliceLoop +popcntOrSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntXorSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntXorSliceEnd +MOVQ m+24(FP), DI +popcntXorSliceLoop: +MOVQ (DI), DX +XORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntXorSliceLoop +popcntXorSliceEnd: +MOVQ AX, ret+48(FP) +RET diff --git a/vendor/github.com/willf/bitset/popcnt_generic.go b/vendor/github.com/willf/bitset/popcnt_generic.go new file mode 100644 index 00000000..6b21cb7a --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_generic.go @@ -0,0 +1,23 @@ +// +build !amd64 appengine + +package bitset + +func popcntSlice(s []uint64) uint64 { + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/xenolf/lego/LICENSE b/vendor/github.com/xenolf/lego/LICENSE index 17460b71..270cba08 100644 --- a/vendor/github.com/xenolf/lego/LICENSE +++ b/vendor/github.com/xenolf/lego/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 Sebastian Erhart +Copyright (c) 2015-2017 Sebastian Erhart Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/github.com/xenolf/lego/acme/client.go b/vendor/github.com/xenolf/lego/acme/client.go index ef0f80b7..bcb84437 100644 --- a/vendor/github.com/xenolf/lego/acme/client.go +++ b/vendor/github.com/xenolf/lego/acme/client.go @@ -330,6 +330,10 @@ DNSNames: challenges, failures := c.getChallenges(domains) // If any challenge fails - return. Do not generate partial SAN certificates. if len(failures) > 0 { + for _, auth := range challenges { + c.disableAuthz(auth) + } + return CertificateResource{}, failures } @@ -373,6 +377,10 @@ func (c *Client) ObtainCertificate(domains []string, bundle bool, privKey crypto challenges, failures := c.getChallenges(domains) // If any challenge fails - return. Do not generate partial SAN certificates. if len(failures) > 0 { + for _, auth := range challenges { + c.disableAuthz(auth) + } + return CertificateResource{}, failures } @@ -493,10 +501,12 @@ func (c *Client) solveChallenges(challenges []authorizationResource) map[string] // TODO: do not immediately fail if one domain fails to validate. err := solver.Solve(authz.Body.Challenges[i], authz.Domain) if err != nil { + c.disableAuthz(authz) failures[authz.Domain] = err } } } else { + c.disableAuthz(authz) failures[authz.Domain] = fmt.Errorf("[%s] acme: Could not determine solvers", authz.Domain) } } @@ -586,6 +596,13 @@ func logAuthz(authz []authorizationResource) { } } +// cleanAuthz loops through the passed in slice and disables any auths which are not "valid" +func (c *Client) disableAuthz(auth authorizationResource) error { + var disabledAuth authorization + _, err := postJSON(c.jws, auth.AuthURL, deactivateAuthMessage{Resource: "authz", Status: "deactivated"}, &disabledAuth) + return err +} + func (c *Client) requestCertificate(authz []authorizationResource, bundle bool, privKey crypto.PrivateKey, mustStaple bool) (CertificateResource, error) { if len(authz) == 0 { return CertificateResource{}, errors.New("Passed no authorizations to requestCertificate!") diff --git a/vendor/github.com/xenolf/lego/acme/dns_challenge.go b/vendor/github.com/xenolf/lego/acme/dns_challenge.go index 30f2170f..d6844dcd 100644 --- a/vendor/github.com/xenolf/lego/acme/dns_challenge.go +++ b/vendor/github.com/xenolf/lego/acme/dns_challenge.go @@ -11,7 +11,6 @@ import ( "time" "github.com/miekg/dns" - "golang.org/x/net/publicsuffix" ) type preCheckDNSFunc func(fqdn, value string) (bool, error) @@ -194,7 +193,7 @@ func dnsQuery(fqdn string, rtype uint16, nameservers []string, recursive bool) ( if err == dns.ErrTruncated { tcp := &dns.Client{Net: "tcp", Timeout: DNSTimeout} - // If the TCP request suceeds, the err will reset to nil + // If the TCP request succeeds, the err will reset to nil in, _, err = tcp.Exchange(m, ns) } @@ -242,10 +241,6 @@ func FindZoneByFqdn(fqdn string, nameservers []string) (string, error) { labelIndexes := dns.Split(fqdn) for _, index := range labelIndexes { domain := fqdn[index:] - // Give up if we have reached the TLD - if isTLD(domain) { - break - } in, err := dnsQuery(domain, dns.TypeSOA, nameservers, true) if err != nil { @@ -260,6 +255,13 @@ func FindZoneByFqdn(fqdn string, nameservers []string) (string, error) { // Check if we got a SOA RR in the answer section if in.Rcode == dns.RcodeSuccess { + + // CNAME records cannot/should not exist at the root of a zone. + // So we skip a domain when a CNAME is found. + if dnsMsgContainsCNAME(in) { + continue + } + for _, ans := range in.Answer { if soa, ok := ans.(*dns.SOA); ok { zone := soa.Hdr.Name @@ -273,10 +275,12 @@ func FindZoneByFqdn(fqdn string, nameservers []string) (string, error) { return "", fmt.Errorf("Could not find the start of authority") } -func isTLD(domain string) bool { - publicsuffix, _ := publicsuffix.PublicSuffix(UnFqdn(domain)) - if publicsuffix == UnFqdn(domain) { - return true +// dnsMsgContainsCNAME checks for a CNAME answer in msg +func dnsMsgContainsCNAME(msg *dns.Msg) bool { + for _, ans := range msg.Answer { + if _, ok := ans.(*dns.CNAME); ok { + return true + } } return false } diff --git a/vendor/github.com/xenolf/lego/acme/http.go b/vendor/github.com/xenolf/lego/acme/http.go index a858b5a7..e469e0de 100644 --- a/vendor/github.com/xenolf/lego/acme/http.go +++ b/vendor/github.com/xenolf/lego/acme/http.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "runtime" "strings" @@ -15,7 +16,18 @@ import ( var UserAgent string // HTTPClient is an HTTP client with a reasonable timeout value. -var HTTPClient = http.Client{Timeout: 10 * time.Second} +var HTTPClient = http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + TLSHandshakeTimeout: 15 * time.Second, + ResponseHeaderTimeout: 15 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, +} const ( // defaultGoUserAgent is the Go HTTP package user agent string. Too diff --git a/vendor/github.com/xenolf/lego/acme/messages.go b/vendor/github.com/xenolf/lego/acme/messages.go index 36db3b21..79ccf154 100644 --- a/vendor/github.com/xenolf/lego/acme/messages.go +++ b/vendor/github.com/xenolf/lego/acme/messages.go @@ -93,6 +93,11 @@ type revokeCertMessage struct { Certificate string `json:"certificate"` } +type deactivateAuthMessage struct { + Resource string `json:"resource,omitempty"` + Status string `jsom:"status"` +} + // CertificateResource represents a CA issued certificate. // PrivateKey, Certificate and IssuerCertificate are all // already PEM encoded and can be directly written to disk. diff --git a/vendor/golang.org/x/crypto/ed25519/ed25519.go b/vendor/golang.org/x/crypto/ed25519/ed25519.go new file mode 100644 index 00000000..a57771a1 --- /dev/null +++ b/vendor/golang.org/x/crypto/ed25519/ed25519.go @@ -0,0 +1,188 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ed25519 implements the Ed25519 signature algorithm. See +// https://ed25519.cr.yp.to/. +// +// These functions are also compatible with the “Ed25519” function defined in +// RFC 8032. +package ed25519 + +// This code is a port of the public domain, “ref10” implementation of ed25519 +// from SUPERCOP. + +import ( + "bytes" + "crypto" + cryptorand "crypto/rand" + "crypto/sha512" + "errors" + "io" + "strconv" + + "golang.org/x/crypto/ed25519/internal/edwards25519" +) + +const ( + // PublicKeySize is the size, in bytes, of public keys as used in this package. + PublicKeySize = 32 + // PrivateKeySize is the size, in bytes, of private keys as used in this package. + PrivateKeySize = 64 + // SignatureSize is the size, in bytes, of signatures generated and verified by this package. + SignatureSize = 64 +) + +// PublicKey is the type of Ed25519 public keys. +type PublicKey []byte + +// PrivateKey is the type of Ed25519 private keys. It implements crypto.Signer. +type PrivateKey []byte + +// Public returns the PublicKey corresponding to priv. +func (priv PrivateKey) Public() crypto.PublicKey { + publicKey := make([]byte, PublicKeySize) + copy(publicKey, priv[32:]) + return PublicKey(publicKey) +} + +// Sign signs the given message with priv. +// Ed25519 performs two passes over messages to be signed and therefore cannot +// handle pre-hashed messages. Thus opts.HashFunc() must return zero to +// indicate the message hasn't been hashed. This can be achieved by passing +// crypto.Hash(0) as the value for opts. +func (priv PrivateKey) Sign(rand io.Reader, message []byte, opts crypto.SignerOpts) (signature []byte, err error) { + if opts.HashFunc() != crypto.Hash(0) { + return nil, errors.New("ed25519: cannot sign hashed message") + } + + return Sign(priv, message), nil +} + +// GenerateKey generates a public/private key pair using entropy from rand. +// If rand is nil, crypto/rand.Reader will be used. +func GenerateKey(rand io.Reader) (publicKey PublicKey, privateKey PrivateKey, err error) { + if rand == nil { + rand = cryptorand.Reader + } + + privateKey = make([]byte, PrivateKeySize) + publicKey = make([]byte, PublicKeySize) + _, err = io.ReadFull(rand, privateKey[:32]) + if err != nil { + return nil, nil, err + } + + digest := sha512.Sum512(privateKey[:32]) + digest[0] &= 248 + digest[31] &= 127 + digest[31] |= 64 + + var A edwards25519.ExtendedGroupElement + var hBytes [32]byte + copy(hBytes[:], digest[:]) + edwards25519.GeScalarMultBase(&A, &hBytes) + var publicKeyBytes [32]byte + A.ToBytes(&publicKeyBytes) + + copy(privateKey[32:], publicKeyBytes[:]) + copy(publicKey, publicKeyBytes[:]) + + return publicKey, privateKey, nil +} + +// Sign signs the message with privateKey and returns a signature. It will +// panic if len(privateKey) is not PrivateKeySize. +func Sign(privateKey PrivateKey, message []byte) []byte { + if l := len(privateKey); l != PrivateKeySize { + panic("ed25519: bad private key length: " + strconv.Itoa(l)) + } + + h := sha512.New() + h.Write(privateKey[:32]) + + var digest1, messageDigest, hramDigest [64]byte + var expandedSecretKey [32]byte + h.Sum(digest1[:0]) + copy(expandedSecretKey[:], digest1[:]) + expandedSecretKey[0] &= 248 + expandedSecretKey[31] &= 63 + expandedSecretKey[31] |= 64 + + h.Reset() + h.Write(digest1[32:]) + h.Write(message) + h.Sum(messageDigest[:0]) + + var messageDigestReduced [32]byte + edwards25519.ScReduce(&messageDigestReduced, &messageDigest) + var R edwards25519.ExtendedGroupElement + edwards25519.GeScalarMultBase(&R, &messageDigestReduced) + + var encodedR [32]byte + R.ToBytes(&encodedR) + + h.Reset() + h.Write(encodedR[:]) + h.Write(privateKey[32:]) + h.Write(message) + h.Sum(hramDigest[:0]) + var hramDigestReduced [32]byte + edwards25519.ScReduce(&hramDigestReduced, &hramDigest) + + var s [32]byte + edwards25519.ScMulAdd(&s, &hramDigestReduced, &expandedSecretKey, &messageDigestReduced) + + signature := make([]byte, SignatureSize) + copy(signature[:], encodedR[:]) + copy(signature[32:], s[:]) + + return signature +} + +// Verify reports whether sig is a valid signature of message by publicKey. It +// will panic if len(publicKey) is not PublicKeySize. +func Verify(publicKey PublicKey, message, sig []byte) bool { + if l := len(publicKey); l != PublicKeySize { + panic("ed25519: bad public key length: " + strconv.Itoa(l)) + } + + if len(sig) != SignatureSize || sig[63]&224 != 0 { + return false + } + + var A edwards25519.ExtendedGroupElement + var publicKeyBytes [32]byte + copy(publicKeyBytes[:], publicKey) + if !A.FromBytes(&publicKeyBytes) { + return false + } + edwards25519.FeNeg(&A.X, &A.X) + edwards25519.FeNeg(&A.T, &A.T) + + h := sha512.New() + h.Write(sig[:32]) + h.Write(publicKey[:]) + h.Write(message) + var digest [64]byte + h.Sum(digest[:0]) + + var hReduced [32]byte + edwards25519.ScReduce(&hReduced, &digest) + + var R edwards25519.ProjectiveGroupElement + var s [32]byte + copy(s[:], sig[32:]) + + // https://tools.ietf.org/html/rfc8032#section-5.1.7 requires that s be in + // the range [0, order) in order to prevent signature malleability. + if !edwards25519.ScMinimal(&s) { + return false + } + + edwards25519.GeDoubleScalarMultVartime(&R, &hReduced, &A, &s) + + var checkR [32]byte + R.ToBytes(&checkR) + return bytes.Equal(sig[:32], checkR[:]) +} diff --git a/vendor/golang.org/x/crypto/ed25519/internal/edwards25519/const.go b/vendor/golang.org/x/crypto/ed25519/internal/edwards25519/const.go new file mode 100644 index 00000000..e39f086c --- /dev/null +++ b/vendor/golang.org/x/crypto/ed25519/internal/edwards25519/const.go @@ -0,0 +1,1422 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package edwards25519 + +// These values are from the public domain, “ref10” implementation of ed25519 +// from SUPERCOP. + +// d is a constant in the Edwards curve equation. +var d = FieldElement{ + -10913610, 13857413, -15372611, 6949391, 114729, -8787816, -6275908, -3247719, -18696448, -12055116, +} + +// d2 is 2*d. +var d2 = FieldElement{ + -21827239, -5839606, -30745221, 13898782, 229458, 15978800, -12551817, -6495438, 29715968, 9444199, +} + +// SqrtM1 is the square-root of -1 in the field. +var SqrtM1 = FieldElement{ + -32595792, -7943725, 9377950, 3500415, 12389472, -272473, -25146209, -2005654, 326686, 11406482, +} + +// A is a constant in the Montgomery-form of curve25519. +var A = FieldElement{ + 486662, 0, 0, 0, 0, 0, 0, 0, 0, 0, +} + +// bi contains precomputed multiples of the base-point. See the Ed25519 paper +// for a discussion about how these values are used. +var bi = [8]PreComputedGroupElement{ + { + FieldElement{25967493, -14356035, 29566456, 3660896, -12694345, 4014787, 27544626, -11754271, -6079156, 2047605}, + FieldElement{-12545711, 934262, -2722910, 3049990, -727428, 9406986, 12720692, 5043384, 19500929, -15469378}, + FieldElement{-8738181, 4489570, 9688441, -14785194, 10184609, -12363380, 29287919, 11864899, -24514362, -4438546}, + }, + { + FieldElement{15636291, -9688557, 24204773, -7912398, 616977, -16685262, 27787600, -14772189, 28944400, -1550024}, + FieldElement{16568933, 4717097, -11556148, -1102322, 15682896, -11807043, 16354577, -11775962, 7689662, 11199574}, + FieldElement{30464156, -5976125, -11779434, -15670865, 23220365, 15915852, 7512774, 10017326, -17749093, -9920357}, + }, + { + FieldElement{10861363, 11473154, 27284546, 1981175, -30064349, 12577861, 32867885, 14515107, -15438304, 10819380}, + FieldElement{4708026, 6336745, 20377586, 9066809, -11272109, 6594696, -25653668, 12483688, -12668491, 5581306}, + FieldElement{19563160, 16186464, -29386857, 4097519, 10237984, -4348115, 28542350, 13850243, -23678021, -15815942}, + }, + { + FieldElement{5153746, 9909285, 1723747, -2777874, 30523605, 5516873, 19480852, 5230134, -23952439, -15175766}, + FieldElement{-30269007, -3463509, 7665486, 10083793, 28475525, 1649722, 20654025, 16520125, 30598449, 7715701}, + FieldElement{28881845, 14381568, 9657904, 3680757, -20181635, 7843316, -31400660, 1370708, 29794553, -1409300}, + }, + { + FieldElement{-22518993, -6692182, 14201702, -8745502, -23510406, 8844726, 18474211, -1361450, -13062696, 13821877}, + FieldElement{-6455177, -7839871, 3374702, -4740862, -27098617, -10571707, 31655028, -7212327, 18853322, -14220951}, + FieldElement{4566830, -12963868, -28974889, -12240689, -7602672, -2830569, -8514358, -10431137, 2207753, -3209784}, + }, + { + FieldElement{-25154831, -4185821, 29681144, 7868801, -6854661, -9423865, -12437364, -663000, -31111463, -16132436}, + FieldElement{25576264, -2703214, 7349804, -11814844, 16472782, 9300885, 3844789, 15725684, 171356, 6466918}, + FieldElement{23103977, 13316479, 9739013, -16149481, 817875, -15038942, 8965339, -14088058, -30714912, 16193877}, + }, + { + FieldElement{-33521811, 3180713, -2394130, 14003687, -16903474, -16270840, 17238398, 4729455, -18074513, 9256800}, + FieldElement{-25182317, -4174131, 32336398, 5036987, -21236817, 11360617, 22616405, 9761698, -19827198, 630305}, + FieldElement{-13720693, 2639453, -24237460, -7406481, 9494427, -5774029, -6554551, -15960994, -2449256, -14291300}, + }, + { + FieldElement{-3151181, -5046075, 9282714, 6866145, -31907062, -863023, -18940575, 15033784, 25105118, -7894876}, + FieldElement{-24326370, 15950226, -31801215, -14592823, -11662737, -5090925, 1573892, -2625887, 2198790, -15804619}, + FieldElement{-3099351, 10324967, -2241613, 7453183, -5446979, -2735503, -13812022, -16236442, -32461234, -12290683}, + }, +} + +// base contains precomputed multiples of the base-point. See the Ed25519 paper +// for a discussion about how these values are used. +var base = [32][8]PreComputedGroupElement{ + { + { + FieldElement{25967493, -14356035, 29566456, 3660896, -12694345, 4014787, 27544626, -11754271, -6079156, 2047605}, + FieldElement{-12545711, 934262, -2722910, 3049990, -727428, 9406986, 12720692, 5043384, 19500929, -15469378}, + FieldElement{-8738181, 4489570, 9688441, -14785194, 10184609, -12363380, 29287919, 11864899, -24514362, -4438546}, + }, + { + FieldElement{-12815894, -12976347, -21581243, 11784320, -25355658, -2750717, -11717903, -3814571, -358445, -10211303}, + FieldElement{-21703237, 6903825, 27185491, 6451973, -29577724, -9554005, -15616551, 11189268, -26829678, -5319081}, + FieldElement{26966642, 11152617, 32442495, 15396054, 14353839, -12752335, -3128826, -9541118, -15472047, -4166697}, + }, + { + FieldElement{15636291, -9688557, 24204773, -7912398, 616977, -16685262, 27787600, -14772189, 28944400, -1550024}, + FieldElement{16568933, 4717097, -11556148, -1102322, 15682896, -11807043, 16354577, -11775962, 7689662, 11199574}, + FieldElement{30464156, -5976125, -11779434, -15670865, 23220365, 15915852, 7512774, 10017326, -17749093, -9920357}, + }, + { + FieldElement{-17036878, 13921892, 10945806, -6033431, 27105052, -16084379, -28926210, 15006023, 3284568, -6276540}, + FieldElement{23599295, -8306047, -11193664, -7687416, 13236774, 10506355, 7464579, 9656445, 13059162, 10374397}, + FieldElement{7798556, 16710257, 3033922, 2874086, 28997861, 2835604, 32406664, -3839045, -641708, -101325}, + }, + { + FieldElement{10861363, 11473154, 27284546, 1981175, -30064349, 12577861, 32867885, 14515107, -15438304, 10819380}, + FieldElement{4708026, 6336745, 20377586, 9066809, -11272109, 6594696, -25653668, 12483688, -12668491, 5581306}, + FieldElement{19563160, 16186464, -29386857, 4097519, 10237984, -4348115, 28542350, 13850243, -23678021, -15815942}, + }, + { + FieldElement{-15371964, -12862754, 32573250, 4720197, -26436522, 5875511, -19188627, -15224819, -9818940, -12085777}, + FieldElement{-8549212, 109983, 15149363, 2178705, 22900618, 4543417, 3044240, -15689887, 1762328, 14866737}, + FieldElement{-18199695, -15951423, -10473290, 1707278, -17185920, 3916101, -28236412, 3959421, 27914454, 4383652}, + }, + { + FieldElement{5153746, 9909285, 1723747, -2777874, 30523605, 5516873, 19480852, 5230134, -23952439, -15175766}, + FieldElement{-30269007, -3463509, 7665486, 10083793, 28475525, 1649722, 20654025, 16520125, 30598449, 7715701}, + FieldElement{28881845, 14381568, 9657904, 3680757, -20181635, 7843316, -31400660, 1370708, 29794553, -1409300}, + }, + { + FieldElement{14499471, -2729599, -33191113, -4254652, 28494862, 14271267, 30290735, 10876454, -33154098, 2381726}, + FieldElement{-7195431, -2655363, -14730155, 462251, -27724326, 3941372, -6236617, 3696005, -32300832, 15351955}, + FieldElement{27431194, 8222322, 16448760, -3907995, -18707002, 11938355, -32961401, -2970515, 29551813, 10109425}, + }, + }, + { + { + FieldElement{-13657040, -13155431, -31283750, 11777098, 21447386, 6519384, -2378284, -1627556, 10092783, -4764171}, + FieldElement{27939166, 14210322, 4677035, 16277044, -22964462, -12398139, -32508754, 12005538, -17810127, 12803510}, + FieldElement{17228999, -15661624, -1233527, 300140, -1224870, -11714777, 30364213, -9038194, 18016357, 4397660}, + }, + { + FieldElement{-10958843, -7690207, 4776341, -14954238, 27850028, -15602212, -26619106, 14544525, -17477504, 982639}, + FieldElement{29253598, 15796703, -2863982, -9908884, 10057023, 3163536, 7332899, -4120128, -21047696, 9934963}, + FieldElement{5793303, 16271923, -24131614, -10116404, 29188560, 1206517, -14747930, 4559895, -30123922, -10897950}, + }, + { + FieldElement{-27643952, -11493006, 16282657, -11036493, 28414021, -15012264, 24191034, 4541697, -13338309, 5500568}, + FieldElement{12650548, -1497113, 9052871, 11355358, -17680037, -8400164, -17430592, 12264343, 10874051, 13524335}, + FieldElement{25556948, -3045990, 714651, 2510400, 23394682, -10415330, 33119038, 5080568, -22528059, 5376628}, + }, + { + FieldElement{-26088264, -4011052, -17013699, -3537628, -6726793, 1920897, -22321305, -9447443, 4535768, 1569007}, + FieldElement{-2255422, 14606630, -21692440, -8039818, 28430649, 8775819, -30494562, 3044290, 31848280, 12543772}, + FieldElement{-22028579, 2943893, -31857513, 6777306, 13784462, -4292203, -27377195, -2062731, 7718482, 14474653}, + }, + { + FieldElement{2385315, 2454213, -22631320, 46603, -4437935, -15680415, 656965, -7236665, 24316168, -5253567}, + FieldElement{13741529, 10911568, -33233417, -8603737, -20177830, -1033297, 33040651, -13424532, -20729456, 8321686}, + FieldElement{21060490, -2212744, 15712757, -4336099, 1639040, 10656336, 23845965, -11874838, -9984458, 608372}, + }, + { + FieldElement{-13672732, -15087586, -10889693, -7557059, -6036909, 11305547, 1123968, -6780577, 27229399, 23887}, + FieldElement{-23244140, -294205, -11744728, 14712571, -29465699, -2029617, 12797024, -6440308, -1633405, 16678954}, + FieldElement{-29500620, 4770662, -16054387, 14001338, 7830047, 9564805, -1508144, -4795045, -17169265, 4904953}, + }, + { + FieldElement{24059557, 14617003, 19037157, -15039908, 19766093, -14906429, 5169211, 16191880, 2128236, -4326833}, + FieldElement{-16981152, 4124966, -8540610, -10653797, 30336522, -14105247, -29806336, 916033, -6882542, -2986532}, + FieldElement{-22630907, 12419372, -7134229, -7473371, -16478904, 16739175, 285431, 2763829, 15736322, 4143876}, + }, + { + FieldElement{2379352, 11839345, -4110402, -5988665, 11274298, 794957, 212801, -14594663, 23527084, -16458268}, + FieldElement{33431127, -11130478, -17838966, -15626900, 8909499, 8376530, -32625340, 4087881, -15188911, -14416214}, + FieldElement{1767683, 7197987, -13205226, -2022635, -13091350, 448826, 5799055, 4357868, -4774191, -16323038}, + }, + }, + { + { + FieldElement{6721966, 13833823, -23523388, -1551314, 26354293, -11863321, 23365147, -3949732, 7390890, 2759800}, + FieldElement{4409041, 2052381, 23373853, 10530217, 7676779, -12885954, 21302353, -4264057, 1244380, -12919645}, + FieldElement{-4421239, 7169619, 4982368, -2957590, 30256825, -2777540, 14086413, 9208236, 15886429, 16489664}, + }, + { + FieldElement{1996075, 10375649, 14346367, 13311202, -6874135, -16438411, -13693198, 398369, -30606455, -712933}, + FieldElement{-25307465, 9795880, -2777414, 14878809, -33531835, 14780363, 13348553, 12076947, -30836462, 5113182}, + FieldElement{-17770784, 11797796, 31950843, 13929123, -25888302, 12288344, -30341101, -7336386, 13847711, 5387222}, + }, + { + FieldElement{-18582163, -3416217, 17824843, -2340966, 22744343, -10442611, 8763061, 3617786, -19600662, 10370991}, + FieldElement{20246567, -14369378, 22358229, -543712, 18507283, -10413996, 14554437, -8746092, 32232924, 16763880}, + FieldElement{9648505, 10094563, 26416693, 14745928, -30374318, -6472621, 11094161, 15689506, 3140038, -16510092}, + }, + { + FieldElement{-16160072, 5472695, 31895588, 4744994, 8823515, 10365685, -27224800, 9448613, -28774454, 366295}, + FieldElement{19153450, 11523972, -11096490, -6503142, -24647631, 5420647, 28344573, 8041113, 719605, 11671788}, + FieldElement{8678025, 2694440, -6808014, 2517372, 4964326, 11152271, -15432916, -15266516, 27000813, -10195553}, + }, + { + FieldElement{-15157904, 7134312, 8639287, -2814877, -7235688, 10421742, 564065, 5336097, 6750977, -14521026}, + FieldElement{11836410, -3979488, 26297894, 16080799, 23455045, 15735944, 1695823, -8819122, 8169720, 16220347}, + FieldElement{-18115838, 8653647, 17578566, -6092619, -8025777, -16012763, -11144307, -2627664, -5990708, -14166033}, + }, + { + FieldElement{-23308498, -10968312, 15213228, -10081214, -30853605, -11050004, 27884329, 2847284, 2655861, 1738395}, + FieldElement{-27537433, -14253021, -25336301, -8002780, -9370762, 8129821, 21651608, -3239336, -19087449, -11005278}, + FieldElement{1533110, 3437855, 23735889, 459276, 29970501, 11335377, 26030092, 5821408, 10478196, 8544890}, + }, + { + FieldElement{32173121, -16129311, 24896207, 3921497, 22579056, -3410854, 19270449, 12217473, 17789017, -3395995}, + FieldElement{-30552961, -2228401, -15578829, -10147201, 13243889, 517024, 15479401, -3853233, 30460520, 1052596}, + FieldElement{-11614875, 13323618, 32618793, 8175907, -15230173, 12596687, 27491595, -4612359, 3179268, -9478891}, + }, + { + FieldElement{31947069, -14366651, -4640583, -15339921, -15125977, -6039709, -14756777, -16411740, 19072640, -9511060}, + FieldElement{11685058, 11822410, 3158003, -13952594, 33402194, -4165066, 5977896, -5215017, 473099, 5040608}, + FieldElement{-20290863, 8198642, -27410132, 11602123, 1290375, -2799760, 28326862, 1721092, -19558642, -3131606}, + }, + }, + { + { + FieldElement{7881532, 10687937, 7578723, 7738378, -18951012, -2553952, 21820786, 8076149, -27868496, 11538389}, + FieldElement{-19935666, 3899861, 18283497, -6801568, -15728660, -11249211, 8754525, 7446702, -5676054, 5797016}, + FieldElement{-11295600, -3793569, -15782110, -7964573, 12708869, -8456199, 2014099, -9050574, -2369172, -5877341}, + }, + { + FieldElement{-22472376, -11568741, -27682020, 1146375, 18956691, 16640559, 1192730, -3714199, 15123619, 10811505}, + FieldElement{14352098, -3419715, -18942044, 10822655, 32750596, 4699007, -70363, 15776356, -28886779, -11974553}, + FieldElement{-28241164, -8072475, -4978962, -5315317, 29416931, 1847569, -20654173, -16484855, 4714547, -9600655}, + }, + { + FieldElement{15200332, 8368572, 19679101, 15970074, -31872674, 1959451, 24611599, -4543832, -11745876, 12340220}, + FieldElement{12876937, -10480056, 33134381, 6590940, -6307776, 14872440, 9613953, 8241152, 15370987, 9608631}, + FieldElement{-4143277, -12014408, 8446281, -391603, 4407738, 13629032, -7724868, 15866074, -28210621, -8814099}, + }, + { + FieldElement{26660628, -15677655, 8393734, 358047, -7401291, 992988, -23904233, 858697, 20571223, 8420556}, + FieldElement{14620715, 13067227, -15447274, 8264467, 14106269, 15080814, 33531827, 12516406, -21574435, -12476749}, + FieldElement{236881, 10476226, 57258, -14677024, 6472998, 2466984, 17258519, 7256740, 8791136, 15069930}, + }, + { + FieldElement{1276410, -9371918, 22949635, -16322807, -23493039, -5702186, 14711875, 4874229, -30663140, -2331391}, + FieldElement{5855666, 4990204, -13711848, 7294284, -7804282, 1924647, -1423175, -7912378, -33069337, 9234253}, + FieldElement{20590503, -9018988, 31529744, -7352666, -2706834, 10650548, 31559055, -11609587, 18979186, 13396066}, + }, + { + FieldElement{24474287, 4968103, 22267082, 4407354, 24063882, -8325180, -18816887, 13594782, 33514650, 7021958}, + FieldElement{-11566906, -6565505, -21365085, 15928892, -26158305, 4315421, -25948728, -3916677, -21480480, 12868082}, + FieldElement{-28635013, 13504661, 19988037, -2132761, 21078225, 6443208, -21446107, 2244500, -12455797, -8089383}, + }, + { + FieldElement{-30595528, 13793479, -5852820, 319136, -25723172, -6263899, 33086546, 8957937, -15233648, 5540521}, + FieldElement{-11630176, -11503902, -8119500, -7643073, 2620056, 1022908, -23710744, -1568984, -16128528, -14962807}, + FieldElement{23152971, 775386, 27395463, 14006635, -9701118, 4649512, 1689819, 892185, -11513277, -15205948}, + }, + { + FieldElement{9770129, 9586738, 26496094, 4324120, 1556511, -3550024, 27453819, 4763127, -19179614, 5867134}, + FieldElement{-32765025, 1927590, 31726409, -4753295, 23962434, -16019500, 27846559, 5931263, -29749703, -16108455}, + FieldElement{27461885, -2977536, 22380810, 1815854, -23033753, -3031938, 7283490, -15148073, -19526700, 7734629}, + }, + }, + { + { + FieldElement{-8010264, -9590817, -11120403, 6196038, 29344158, -13430885, 7585295, -3176626, 18549497, 15302069}, + FieldElement{-32658337, -6171222, -7672793, -11051681, 6258878, 13504381, 10458790, -6418461, -8872242, 8424746}, + FieldElement{24687205, 8613276, -30667046, -3233545, 1863892, -1830544, 19206234, 7134917, -11284482, -828919}, + }, + { + FieldElement{11334899, -9218022, 8025293, 12707519, 17523892, -10476071, 10243738, -14685461, -5066034, 16498837}, + FieldElement{8911542, 6887158, -9584260, -6958590, 11145641, -9543680, 17303925, -14124238, 6536641, 10543906}, + FieldElement{-28946384, 15479763, -17466835, 568876, -1497683, 11223454, -2669190, -16625574, -27235709, 8876771}, + }, + { + FieldElement{-25742899, -12566864, -15649966, -846607, -33026686, -796288, -33481822, 15824474, -604426, -9039817}, + FieldElement{10330056, 70051, 7957388, -9002667, 9764902, 15609756, 27698697, -4890037, 1657394, 3084098}, + FieldElement{10477963, -7470260, 12119566, -13250805, 29016247, -5365589, 31280319, 14396151, -30233575, 15272409}, + }, + { + FieldElement{-12288309, 3169463, 28813183, 16658753, 25116432, -5630466, -25173957, -12636138, -25014757, 1950504}, + FieldElement{-26180358, 9489187, 11053416, -14746161, -31053720, 5825630, -8384306, -8767532, 15341279, 8373727}, + FieldElement{28685821, 7759505, -14378516, -12002860, -31971820, 4079242, 298136, -10232602, -2878207, 15190420}, + }, + { + FieldElement{-32932876, 13806336, -14337485, -15794431, -24004620, 10940928, 8669718, 2742393, -26033313, -6875003}, + FieldElement{-1580388, -11729417, -25979658, -11445023, -17411874, -10912854, 9291594, -16247779, -12154742, 6048605}, + FieldElement{-30305315, 14843444, 1539301, 11864366, 20201677, 1900163, 13934231, 5128323, 11213262, 9168384}, + }, + { + FieldElement{-26280513, 11007847, 19408960, -940758, -18592965, -4328580, -5088060, -11105150, 20470157, -16398701}, + FieldElement{-23136053, 9282192, 14855179, -15390078, -7362815, -14408560, -22783952, 14461608, 14042978, 5230683}, + FieldElement{29969567, -2741594, -16711867, -8552442, 9175486, -2468974, 21556951, 3506042, -5933891, -12449708}, + }, + { + FieldElement{-3144746, 8744661, 19704003, 4581278, -20430686, 6830683, -21284170, 8971513, -28539189, 15326563}, + FieldElement{-19464629, 10110288, -17262528, -3503892, -23500387, 1355669, -15523050, 15300988, -20514118, 9168260}, + FieldElement{-5353335, 4488613, -23803248, 16314347, 7780487, -15638939, -28948358, 9601605, 33087103, -9011387}, + }, + { + FieldElement{-19443170, -15512900, -20797467, -12445323, -29824447, 10229461, -27444329, -15000531, -5996870, 15664672}, + FieldElement{23294591, -16632613, -22650781, -8470978, 27844204, 11461195, 13099750, -2460356, 18151676, 13417686}, + FieldElement{-24722913, -4176517, -31150679, 5988919, -26858785, 6685065, 1661597, -12551441, 15271676, -15452665}, + }, + }, + { + { + FieldElement{11433042, -13228665, 8239631, -5279517, -1985436, -725718, -18698764, 2167544, -6921301, -13440182}, + FieldElement{-31436171, 15575146, 30436815, 12192228, -22463353, 9395379, -9917708, -8638997, 12215110, 12028277}, + FieldElement{14098400, 6555944, 23007258, 5757252, -15427832, -12950502, 30123440, 4617780, -16900089, -655628}, + }, + { + FieldElement{-4026201, -15240835, 11893168, 13718664, -14809462, 1847385, -15819999, 10154009, 23973261, -12684474}, + FieldElement{-26531820, -3695990, -1908898, 2534301, -31870557, -16550355, 18341390, -11419951, 32013174, -10103539}, + FieldElement{-25479301, 10876443, -11771086, -14625140, -12369567, 1838104, 21911214, 6354752, 4425632, -837822}, + }, + { + FieldElement{-10433389, -14612966, 22229858, -3091047, -13191166, 776729, -17415375, -12020462, 4725005, 14044970}, + FieldElement{19268650, -7304421, 1555349, 8692754, -21474059, -9910664, 6347390, -1411784, -19522291, -16109756}, + FieldElement{-24864089, 12986008, -10898878, -5558584, -11312371, -148526, 19541418, 8180106, 9282262, 10282508}, + }, + { + FieldElement{-26205082, 4428547, -8661196, -13194263, 4098402, -14165257, 15522535, 8372215, 5542595, -10702683}, + FieldElement{-10562541, 14895633, 26814552, -16673850, -17480754, -2489360, -2781891, 6993761, -18093885, 10114655}, + FieldElement{-20107055, -929418, 31422704, 10427861, -7110749, 6150669, -29091755, -11529146, 25953725, -106158}, + }, + { + FieldElement{-4234397, -8039292, -9119125, 3046000, 2101609, -12607294, 19390020, 6094296, -3315279, 12831125}, + FieldElement{-15998678, 7578152, 5310217, 14408357, -33548620, -224739, 31575954, 6326196, 7381791, -2421839}, + FieldElement{-20902779, 3296811, 24736065, -16328389, 18374254, 7318640, 6295303, 8082724, -15362489, 12339664}, + }, + { + FieldElement{27724736, 2291157, 6088201, -14184798, 1792727, 5857634, 13848414, 15768922, 25091167, 14856294}, + FieldElement{-18866652, 8331043, 24373479, 8541013, -701998, -9269457, 12927300, -12695493, -22182473, -9012899}, + FieldElement{-11423429, -5421590, 11632845, 3405020, 30536730, -11674039, -27260765, 13866390, 30146206, 9142070}, + }, + { + FieldElement{3924129, -15307516, -13817122, -10054960, 12291820, -668366, -27702774, 9326384, -8237858, 4171294}, + FieldElement{-15921940, 16037937, 6713787, 16606682, -21612135, 2790944, 26396185, 3731949, 345228, -5462949}, + FieldElement{-21327538, 13448259, 25284571, 1143661, 20614966, -8849387, 2031539, -12391231, -16253183, -13582083}, + }, + { + FieldElement{31016211, -16722429, 26371392, -14451233, -5027349, 14854137, 17477601, 3842657, 28012650, -16405420}, + FieldElement{-5075835, 9368966, -8562079, -4600902, -15249953, 6970560, -9189873, 16292057, -8867157, 3507940}, + FieldElement{29439664, 3537914, 23333589, 6997794, -17555561, -11018068, -15209202, -15051267, -9164929, 6580396}, + }, + }, + { + { + FieldElement{-12185861, -7679788, 16438269, 10826160, -8696817, -6235611, 17860444, -9273846, -2095802, 9304567}, + FieldElement{20714564, -4336911, 29088195, 7406487, 11426967, -5095705, 14792667, -14608617, 5289421, -477127}, + FieldElement{-16665533, -10650790, -6160345, -13305760, 9192020, -1802462, 17271490, 12349094, 26939669, -3752294}, + }, + { + FieldElement{-12889898, 9373458, 31595848, 16374215, 21471720, 13221525, -27283495, -12348559, -3698806, 117887}, + FieldElement{22263325, -6560050, 3984570, -11174646, -15114008, -566785, 28311253, 5358056, -23319780, 541964}, + FieldElement{16259219, 3261970, 2309254, -15534474, -16885711, -4581916, 24134070, -16705829, -13337066, -13552195}, + }, + { + FieldElement{9378160, -13140186, -22845982, -12745264, 28198281, -7244098, -2399684, -717351, 690426, 14876244}, + FieldElement{24977353, -314384, -8223969, -13465086, 28432343, -1176353, -13068804, -12297348, -22380984, 6618999}, + FieldElement{-1538174, 11685646, 12944378, 13682314, -24389511, -14413193, 8044829, -13817328, 32239829, -5652762}, + }, + { + FieldElement{-18603066, 4762990, -926250, 8885304, -28412480, -3187315, 9781647, -10350059, 32779359, 5095274}, + FieldElement{-33008130, -5214506, -32264887, -3685216, 9460461, -9327423, -24601656, 14506724, 21639561, -2630236}, + FieldElement{-16400943, -13112215, 25239338, 15531969, 3987758, -4499318, -1289502, -6863535, 17874574, 558605}, + }, + { + FieldElement{-13600129, 10240081, 9171883, 16131053, -20869254, 9599700, 33499487, 5080151, 2085892, 5119761}, + FieldElement{-22205145, -2519528, -16381601, 414691, -25019550, 2170430, 30634760, -8363614, -31999993, -5759884}, + FieldElement{-6845704, 15791202, 8550074, -1312654, 29928809, -12092256, 27534430, -7192145, -22351378, 12961482}, + }, + { + FieldElement{-24492060, -9570771, 10368194, 11582341, -23397293, -2245287, 16533930, 8206996, -30194652, -5159638}, + FieldElement{-11121496, -3382234, 2307366, 6362031, -135455, 8868177, -16835630, 7031275, 7589640, 8945490}, + FieldElement{-32152748, 8917967, 6661220, -11677616, -1192060, -15793393, 7251489, -11182180, 24099109, -14456170}, + }, + { + FieldElement{5019558, -7907470, 4244127, -14714356, -26933272, 6453165, -19118182, -13289025, -6231896, -10280736}, + FieldElement{10853594, 10721687, 26480089, 5861829, -22995819, 1972175, -1866647, -10557898, -3363451, -6441124}, + FieldElement{-17002408, 5906790, 221599, -6563147, 7828208, -13248918, 24362661, -2008168, -13866408, 7421392}, + }, + { + FieldElement{8139927, -6546497, 32257646, -5890546, 30375719, 1886181, -21175108, 15441252, 28826358, -4123029}, + FieldElement{6267086, 9695052, 7709135, -16603597, -32869068, -1886135, 14795160, -7840124, 13746021, -1742048}, + FieldElement{28584902, 7787108, -6732942, -15050729, 22846041, -7571236, -3181936, -363524, 4771362, -8419958}, + }, + }, + { + { + FieldElement{24949256, 6376279, -27466481, -8174608, -18646154, -9930606, 33543569, -12141695, 3569627, 11342593}, + FieldElement{26514989, 4740088, 27912651, 3697550, 19331575, -11472339, 6809886, 4608608, 7325975, -14801071}, + FieldElement{-11618399, -14554430, -24321212, 7655128, -1369274, 5214312, -27400540, 10258390, -17646694, -8186692}, + }, + { + FieldElement{11431204, 15823007, 26570245, 14329124, 18029990, 4796082, -31446179, 15580664, 9280358, -3973687}, + FieldElement{-160783, -10326257, -22855316, -4304997, -20861367, -13621002, -32810901, -11181622, -15545091, 4387441}, + FieldElement{-20799378, 12194512, 3937617, -5805892, -27154820, 9340370, -24513992, 8548137, 20617071, -7482001}, + }, + { + FieldElement{-938825, -3930586, -8714311, 16124718, 24603125, -6225393, -13775352, -11875822, 24345683, 10325460}, + FieldElement{-19855277, -1568885, -22202708, 8714034, 14007766, 6928528, 16318175, -1010689, 4766743, 3552007}, + FieldElement{-21751364, -16730916, 1351763, -803421, -4009670, 3950935, 3217514, 14481909, 10988822, -3994762}, + }, + { + FieldElement{15564307, -14311570, 3101243, 5684148, 30446780, -8051356, 12677127, -6505343, -8295852, 13296005}, + FieldElement{-9442290, 6624296, -30298964, -11913677, -4670981, -2057379, 31521204, 9614054, -30000824, 12074674}, + FieldElement{4771191, -135239, 14290749, -13089852, 27992298, 14998318, -1413936, -1556716, 29832613, -16391035}, + }, + { + FieldElement{7064884, -7541174, -19161962, -5067537, -18891269, -2912736, 25825242, 5293297, -27122660, 13101590}, + FieldElement{-2298563, 2439670, -7466610, 1719965, -27267541, -16328445, 32512469, -5317593, -30356070, -4190957}, + FieldElement{-30006540, 10162316, -33180176, 3981723, -16482138, -13070044, 14413974, 9515896, 19568978, 9628812}, + }, + { + FieldElement{33053803, 199357, 15894591, 1583059, 27380243, -4580435, -17838894, -6106839, -6291786, 3437740}, + FieldElement{-18978877, 3884493, 19469877, 12726490, 15913552, 13614290, -22961733, 70104, 7463304, 4176122}, + FieldElement{-27124001, 10659917, 11482427, -16070381, 12771467, -6635117, -32719404, -5322751, 24216882, 5944158}, + }, + { + FieldElement{8894125, 7450974, -2664149, -9765752, -28080517, -12389115, 19345746, 14680796, 11632993, 5847885}, + FieldElement{26942781, -2315317, 9129564, -4906607, 26024105, 11769399, -11518837, 6367194, -9727230, 4782140}, + FieldElement{19916461, -4828410, -22910704, -11414391, 25606324, -5972441, 33253853, 8220911, 6358847, -1873857}, + }, + { + FieldElement{801428, -2081702, 16569428, 11065167, 29875704, 96627, 7908388, -4480480, -13538503, 1387155}, + FieldElement{19646058, 5720633, -11416706, 12814209, 11607948, 12749789, 14147075, 15156355, -21866831, 11835260}, + FieldElement{19299512, 1155910, 28703737, 14890794, 2925026, 7269399, 26121523, 15467869, -26560550, 5052483}, + }, + }, + { + { + FieldElement{-3017432, 10058206, 1980837, 3964243, 22160966, 12322533, -6431123, -12618185, 12228557, -7003677}, + FieldElement{32944382, 14922211, -22844894, 5188528, 21913450, -8719943, 4001465, 13238564, -6114803, 8653815}, + FieldElement{22865569, -4652735, 27603668, -12545395, 14348958, 8234005, 24808405, 5719875, 28483275, 2841751}, + }, + { + FieldElement{-16420968, -1113305, -327719, -12107856, 21886282, -15552774, -1887966, -315658, 19932058, -12739203}, + FieldElement{-11656086, 10087521, -8864888, -5536143, -19278573, -3055912, 3999228, 13239134, -4777469, -13910208}, + FieldElement{1382174, -11694719, 17266790, 9194690, -13324356, 9720081, 20403944, 11284705, -14013818, 3093230}, + }, + { + FieldElement{16650921, -11037932, -1064178, 1570629, -8329746, 7352753, -302424, 16271225, -24049421, -6691850}, + FieldElement{-21911077, -5927941, -4611316, -5560156, -31744103, -10785293, 24123614, 15193618, -21652117, -16739389}, + FieldElement{-9935934, -4289447, -25279823, 4372842, 2087473, 10399484, 31870908, 14690798, 17361620, 11864968}, + }, + { + FieldElement{-11307610, 6210372, 13206574, 5806320, -29017692, -13967200, -12331205, -7486601, -25578460, -16240689}, + FieldElement{14668462, -12270235, 26039039, 15305210, 25515617, 4542480, 10453892, 6577524, 9145645, -6443880}, + FieldElement{5974874, 3053895, -9433049, -10385191, -31865124, 3225009, -7972642, 3936128, -5652273, -3050304}, + }, + { + FieldElement{30625386, -4729400, -25555961, -12792866, -20484575, 7695099, 17097188, -16303496, -27999779, 1803632}, + FieldElement{-3553091, 9865099, -5228566, 4272701, -5673832, -16689700, 14911344, 12196514, -21405489, 7047412}, + FieldElement{20093277, 9920966, -11138194, -5343857, 13161587, 12044805, -32856851, 4124601, -32343828, -10257566}, + }, + { + FieldElement{-20788824, 14084654, -13531713, 7842147, 19119038, -13822605, 4752377, -8714640, -21679658, 2288038}, + FieldElement{-26819236, -3283715, 29965059, 3039786, -14473765, 2540457, 29457502, 14625692, -24819617, 12570232}, + FieldElement{-1063558, -11551823, 16920318, 12494842, 1278292, -5869109, -21159943, -3498680, -11974704, 4724943}, + }, + { + FieldElement{17960970, -11775534, -4140968, -9702530, -8876562, -1410617, -12907383, -8659932, -29576300, 1903856}, + FieldElement{23134274, -14279132, -10681997, -1611936, 20684485, 15770816, -12989750, 3190296, 26955097, 14109738}, + FieldElement{15308788, 5320727, -30113809, -14318877, 22902008, 7767164, 29425325, -11277562, 31960942, 11934971}, + }, + { + FieldElement{-27395711, 8435796, 4109644, 12222639, -24627868, 14818669, 20638173, 4875028, 10491392, 1379718}, + FieldElement{-13159415, 9197841, 3875503, -8936108, -1383712, -5879801, 33518459, 16176658, 21432314, 12180697}, + FieldElement{-11787308, 11500838, 13787581, -13832590, -22430679, 10140205, 1465425, 12689540, -10301319, -13872883}, + }, + }, + { + { + FieldElement{5414091, -15386041, -21007664, 9643570, 12834970, 1186149, -2622916, -1342231, 26128231, 6032912}, + FieldElement{-26337395, -13766162, 32496025, -13653919, 17847801, -12669156, 3604025, 8316894, -25875034, -10437358}, + FieldElement{3296484, 6223048, 24680646, -12246460, -23052020, 5903205, -8862297, -4639164, 12376617, 3188849}, + }, + { + FieldElement{29190488, -14659046, 27549113, -1183516, 3520066, -10697301, 32049515, -7309113, -16109234, -9852307}, + FieldElement{-14744486, -9309156, 735818, -598978, -20407687, -5057904, 25246078, -15795669, 18640741, -960977}, + FieldElement{-6928835, -16430795, 10361374, 5642961, 4910474, 12345252, -31638386, -494430, 10530747, 1053335}, + }, + { + FieldElement{-29265967, -14186805, -13538216, -12117373, -19457059, -10655384, -31462369, -2948985, 24018831, 15026644}, + FieldElement{-22592535, -3145277, -2289276, 5953843, -13440189, 9425631, 25310643, 13003497, -2314791, -15145616}, + FieldElement{-27419985, -603321, -8043984, -1669117, -26092265, 13987819, -27297622, 187899, -23166419, -2531735}, + }, + { + FieldElement{-21744398, -13810475, 1844840, 5021428, -10434399, -15911473, 9716667, 16266922, -5070217, 726099}, + FieldElement{29370922, -6053998, 7334071, -15342259, 9385287, 2247707, -13661962, -4839461, 30007388, -15823341}, + FieldElement{-936379, 16086691, 23751945, -543318, -1167538, -5189036, 9137109, 730663, 9835848, 4555336}, + }, + { + FieldElement{-23376435, 1410446, -22253753, -12899614, 30867635, 15826977, 17693930, 544696, -11985298, 12422646}, + FieldElement{31117226, -12215734, -13502838, 6561947, -9876867, -12757670, -5118685, -4096706, 29120153, 13924425}, + FieldElement{-17400879, -14233209, 19675799, -2734756, -11006962, -5858820, -9383939, -11317700, 7240931, -237388}, + }, + { + FieldElement{-31361739, -11346780, -15007447, -5856218, -22453340, -12152771, 1222336, 4389483, 3293637, -15551743}, + FieldElement{-16684801, -14444245, 11038544, 11054958, -13801175, -3338533, -24319580, 7733547, 12796905, -6335822}, + FieldElement{-8759414, -10817836, -25418864, 10783769, -30615557, -9746811, -28253339, 3647836, 3222231, -11160462}, + }, + { + FieldElement{18606113, 1693100, -25448386, -15170272, 4112353, 10045021, 23603893, -2048234, -7550776, 2484985}, + FieldElement{9255317, -3131197, -12156162, -1004256, 13098013, -9214866, 16377220, -2102812, -19802075, -3034702}, + FieldElement{-22729289, 7496160, -5742199, 11329249, 19991973, -3347502, -31718148, 9936966, -30097688, -10618797}, + }, + { + FieldElement{21878590, -5001297, 4338336, 13643897, -3036865, 13160960, 19708896, 5415497, -7360503, -4109293}, + FieldElement{27736861, 10103576, 12500508, 8502413, -3413016, -9633558, 10436918, -1550276, -23659143, -8132100}, + FieldElement{19492550, -12104365, -29681976, -852630, -3208171, 12403437, 30066266, 8367329, 13243957, 8709688}, + }, + }, + { + { + FieldElement{12015105, 2801261, 28198131, 10151021, 24818120, -4743133, -11194191, -5645734, 5150968, 7274186}, + FieldElement{2831366, -12492146, 1478975, 6122054, 23825128, -12733586, 31097299, 6083058, 31021603, -9793610}, + FieldElement{-2529932, -2229646, 445613, 10720828, -13849527, -11505937, -23507731, 16354465, 15067285, -14147707}, + }, + { + FieldElement{7840942, 14037873, -33364863, 15934016, -728213, -3642706, 21403988, 1057586, -19379462, -12403220}, + FieldElement{915865, -16469274, 15608285, -8789130, -24357026, 6060030, -17371319, 8410997, -7220461, 16527025}, + FieldElement{32922597, -556987, 20336074, -16184568, 10903705, -5384487, 16957574, 52992, 23834301, 6588044}, + }, + { + FieldElement{32752030, 11232950, 3381995, -8714866, 22652988, -10744103, 17159699, 16689107, -20314580, -1305992}, + FieldElement{-4689649, 9166776, -25710296, -10847306, 11576752, 12733943, 7924251, -2752281, 1976123, -7249027}, + FieldElement{21251222, 16309901, -2983015, -6783122, 30810597, 12967303, 156041, -3371252, 12331345, -8237197}, + }, + { + FieldElement{8651614, -4477032, -16085636, -4996994, 13002507, 2950805, 29054427, -5106970, 10008136, -4667901}, + FieldElement{31486080, 15114593, -14261250, 12951354, 14369431, -7387845, 16347321, -13662089, 8684155, -10532952}, + FieldElement{19443825, 11385320, 24468943, -9659068, -23919258, 2187569, -26263207, -6086921, 31316348, 14219878}, + }, + { + FieldElement{-28594490, 1193785, 32245219, 11392485, 31092169, 15722801, 27146014, 6992409, 29126555, 9207390}, + FieldElement{32382935, 1110093, 18477781, 11028262, -27411763, -7548111, -4980517, 10843782, -7957600, -14435730}, + FieldElement{2814918, 7836403, 27519878, -7868156, -20894015, -11553689, -21494559, 8550130, 28346258, 1994730}, + }, + { + FieldElement{-19578299, 8085545, -14000519, -3948622, 2785838, -16231307, -19516951, 7174894, 22628102, 8115180}, + FieldElement{-30405132, 955511, -11133838, -15078069, -32447087, -13278079, -25651578, 3317160, -9943017, 930272}, + FieldElement{-15303681, -6833769, 28856490, 1357446, 23421993, 1057177, 24091212, -1388970, -22765376, -10650715}, + }, + { + FieldElement{-22751231, -5303997, -12907607, -12768866, -15811511, -7797053, -14839018, -16554220, -1867018, 8398970}, + FieldElement{-31969310, 2106403, -4736360, 1362501, 12813763, 16200670, 22981545, -6291273, 18009408, -15772772}, + FieldElement{-17220923, -9545221, -27784654, 14166835, 29815394, 7444469, 29551787, -3727419, 19288549, 1325865}, + }, + { + FieldElement{15100157, -15835752, -23923978, -1005098, -26450192, 15509408, 12376730, -3479146, 33166107, -8042750}, + FieldElement{20909231, 13023121, -9209752, 16251778, -5778415, -8094914, 12412151, 10018715, 2213263, -13878373}, + FieldElement{32529814, -11074689, 30361439, -16689753, -9135940, 1513226, 22922121, 6382134, -5766928, 8371348}, + }, + }, + { + { + FieldElement{9923462, 11271500, 12616794, 3544722, -29998368, -1721626, 12891687, -8193132, -26442943, 10486144}, + FieldElement{-22597207, -7012665, 8587003, -8257861, 4084309, -12970062, 361726, 2610596, -23921530, -11455195}, + FieldElement{5408411, -1136691, -4969122, 10561668, 24145918, 14240566, 31319731, -4235541, 19985175, -3436086}, + }, + { + FieldElement{-13994457, 16616821, 14549246, 3341099, 32155958, 13648976, -17577068, 8849297, 65030, 8370684}, + FieldElement{-8320926, -12049626, 31204563, 5839400, -20627288, -1057277, -19442942, 6922164, 12743482, -9800518}, + FieldElement{-2361371, 12678785, 28815050, 4759974, -23893047, 4884717, 23783145, 11038569, 18800704, 255233}, + }, + { + FieldElement{-5269658, -1773886, 13957886, 7990715, 23132995, 728773, 13393847, 9066957, 19258688, -14753793}, + FieldElement{-2936654, -10827535, -10432089, 14516793, -3640786, 4372541, -31934921, 2209390, -1524053, 2055794}, + FieldElement{580882, 16705327, 5468415, -2683018, -30926419, -14696000, -7203346, -8994389, -30021019, 7394435}, + }, + { + FieldElement{23838809, 1822728, -15738443, 15242727, 8318092, -3733104, -21672180, -3492205, -4821741, 14799921}, + FieldElement{13345610, 9759151, 3371034, -16137791, 16353039, 8577942, 31129804, 13496856, -9056018, 7402518}, + FieldElement{2286874, -4435931, -20042458, -2008336, -13696227, 5038122, 11006906, -15760352, 8205061, 1607563}, + }, + { + FieldElement{14414086, -8002132, 3331830, -3208217, 22249151, -5594188, 18364661, -2906958, 30019587, -9029278}, + FieldElement{-27688051, 1585953, -10775053, 931069, -29120221, -11002319, -14410829, 12029093, 9944378, 8024}, + FieldElement{4368715, -3709630, 29874200, -15022983, -20230386, -11410704, -16114594, -999085, -8142388, 5640030}, + }, + { + FieldElement{10299610, 13746483, 11661824, 16234854, 7630238, 5998374, 9809887, -16694564, 15219798, -14327783}, + FieldElement{27425505, -5719081, 3055006, 10660664, 23458024, 595578, -15398605, -1173195, -18342183, 9742717}, + FieldElement{6744077, 2427284, 26042789, 2720740, -847906, 1118974, 32324614, 7406442, 12420155, 1994844}, + }, + { + FieldElement{14012521, -5024720, -18384453, -9578469, -26485342, -3936439, -13033478, -10909803, 24319929, -6446333}, + FieldElement{16412690, -4507367, 10772641, 15929391, -17068788, -4658621, 10555945, -10484049, -30102368, -4739048}, + FieldElement{22397382, -7767684, -9293161, -12792868, 17166287, -9755136, -27333065, 6199366, 21880021, -12250760}, + }, + { + FieldElement{-4283307, 5368523, -31117018, 8163389, -30323063, 3209128, 16557151, 8890729, 8840445, 4957760}, + FieldElement{-15447727, 709327, -6919446, -10870178, -29777922, 6522332, -21720181, 12130072, -14796503, 5005757}, + FieldElement{-2114751, -14308128, 23019042, 15765735, -25269683, 6002752, 10183197, -13239326, -16395286, -2176112}, + }, + }, + { + { + FieldElement{-19025756, 1632005, 13466291, -7995100, -23640451, 16573537, -32013908, -3057104, 22208662, 2000468}, + FieldElement{3065073, -1412761, -25598674, -361432, -17683065, -5703415, -8164212, 11248527, -3691214, -7414184}, + FieldElement{10379208, -6045554, 8877319, 1473647, -29291284, -12507580, 16690915, 2553332, -3132688, 16400289}, + }, + { + FieldElement{15716668, 1254266, -18472690, 7446274, -8448918, 6344164, -22097271, -7285580, 26894937, 9132066}, + FieldElement{24158887, 12938817, 11085297, -8177598, -28063478, -4457083, -30576463, 64452, -6817084, -2692882}, + FieldElement{13488534, 7794716, 22236231, 5989356, 25426474, -12578208, 2350710, -3418511, -4688006, 2364226}, + }, + { + FieldElement{16335052, 9132434, 25640582, 6678888, 1725628, 8517937, -11807024, -11697457, 15445875, -7798101}, + FieldElement{29004207, -7867081, 28661402, -640412, -12794003, -7943086, 31863255, -4135540, -278050, -15759279}, + FieldElement{-6122061, -14866665, -28614905, 14569919, -10857999, -3591829, 10343412, -6976290, -29828287, -10815811}, + }, + { + FieldElement{27081650, 3463984, 14099042, -4517604, 1616303, -6205604, 29542636, 15372179, 17293797, 960709}, + FieldElement{20263915, 11434237, -5765435, 11236810, 13505955, -10857102, -16111345, 6493122, -19384511, 7639714}, + FieldElement{-2830798, -14839232, 25403038, -8215196, -8317012, -16173699, 18006287, -16043750, 29994677, -15808121}, + }, + { + FieldElement{9769828, 5202651, -24157398, -13631392, -28051003, -11561624, -24613141, -13860782, -31184575, 709464}, + FieldElement{12286395, 13076066, -21775189, -1176622, -25003198, 4057652, -32018128, -8890874, 16102007, 13205847}, + FieldElement{13733362, 5599946, 10557076, 3195751, -5557991, 8536970, -25540170, 8525972, 10151379, 10394400}, + }, + { + FieldElement{4024660, -16137551, 22436262, 12276534, -9099015, -2686099, 19698229, 11743039, -33302334, 8934414}, + FieldElement{-15879800, -4525240, -8580747, -2934061, 14634845, -698278, -9449077, 3137094, -11536886, 11721158}, + FieldElement{17555939, -5013938, 8268606, 2331751, -22738815, 9761013, 9319229, 8835153, -9205489, -1280045}, + }, + { + FieldElement{-461409, -7830014, 20614118, 16688288, -7514766, -4807119, 22300304, 505429, 6108462, -6183415}, + FieldElement{-5070281, 12367917, -30663534, 3234473, 32617080, -8422642, 29880583, -13483331, -26898490, -7867459}, + FieldElement{-31975283, 5726539, 26934134, 10237677, -3173717, -605053, 24199304, 3795095, 7592688, -14992079}, + }, + { + FieldElement{21594432, -14964228, 17466408, -4077222, 32537084, 2739898, 6407723, 12018833, -28256052, 4298412}, + FieldElement{-20650503, -11961496, -27236275, 570498, 3767144, -1717540, 13891942, -1569194, 13717174, 10805743}, + FieldElement{-14676630, -15644296, 15287174, 11927123, 24177847, -8175568, -796431, 14860609, -26938930, -5863836}, + }, + }, + { + { + FieldElement{12962541, 5311799, -10060768, 11658280, 18855286, -7954201, 13286263, -12808704, -4381056, 9882022}, + FieldElement{18512079, 11319350, -20123124, 15090309, 18818594, 5271736, -22727904, 3666879, -23967430, -3299429}, + FieldElement{-6789020, -3146043, 16192429, 13241070, 15898607, -14206114, -10084880, -6661110, -2403099, 5276065}, + }, + { + FieldElement{30169808, -5317648, 26306206, -11750859, 27814964, 7069267, 7152851, 3684982, 1449224, 13082861}, + FieldElement{10342826, 3098505, 2119311, 193222, 25702612, 12233820, 23697382, 15056736, -21016438, -8202000}, + FieldElement{-33150110, 3261608, 22745853, 7948688, 19370557, -15177665, -26171976, 6482814, -10300080, -11060101}, + }, + { + FieldElement{32869458, -5408545, 25609743, 15678670, -10687769, -15471071, 26112421, 2521008, -22664288, 6904815}, + FieldElement{29506923, 4457497, 3377935, -9796444, -30510046, 12935080, 1561737, 3841096, -29003639, -6657642}, + FieldElement{10340844, -6630377, -18656632, -2278430, 12621151, -13339055, 30878497, -11824370, -25584551, 5181966}, + }, + { + FieldElement{25940115, -12658025, 17324188, -10307374, -8671468, 15029094, 24396252, -16450922, -2322852, -12388574}, + FieldElement{-21765684, 9916823, -1300409, 4079498, -1028346, 11909559, 1782390, 12641087, 20603771, -6561742}, + FieldElement{-18882287, -11673380, 24849422, 11501709, 13161720, -4768874, 1925523, 11914390, 4662781, 7820689}, + }, + { + FieldElement{12241050, -425982, 8132691, 9393934, 32846760, -1599620, 29749456, 12172924, 16136752, 15264020}, + FieldElement{-10349955, -14680563, -8211979, 2330220, -17662549, -14545780, 10658213, 6671822, 19012087, 3772772}, + FieldElement{3753511, -3421066, 10617074, 2028709, 14841030, -6721664, 28718732, -15762884, 20527771, 12988982}, + }, + { + FieldElement{-14822485, -5797269, -3707987, 12689773, -898983, -10914866, -24183046, -10564943, 3299665, -12424953}, + FieldElement{-16777703, -15253301, -9642417, 4978983, 3308785, 8755439, 6943197, 6461331, -25583147, 8991218}, + FieldElement{-17226263, 1816362, -1673288, -6086439, 31783888, -8175991, -32948145, 7417950, -30242287, 1507265}, + }, + { + FieldElement{29692663, 6829891, -10498800, 4334896, 20945975, -11906496, -28887608, 8209391, 14606362, -10647073}, + FieldElement{-3481570, 8707081, 32188102, 5672294, 22096700, 1711240, -33020695, 9761487, 4170404, -2085325}, + FieldElement{-11587470, 14855945, -4127778, -1531857, -26649089, 15084046, 22186522, 16002000, -14276837, -8400798}, + }, + { + FieldElement{-4811456, 13761029, -31703877, -2483919, -3312471, 7869047, -7113572, -9620092, 13240845, 10965870}, + FieldElement{-7742563, -8256762, -14768334, -13656260, -23232383, 12387166, 4498947, 14147411, 29514390, 4302863}, + FieldElement{-13413405, -12407859, 20757302, -13801832, 14785143, 8976368, -5061276, -2144373, 17846988, -13971927}, + }, + }, + { + { + FieldElement{-2244452, -754728, -4597030, -1066309, -6247172, 1455299, -21647728, -9214789, -5222701, 12650267}, + FieldElement{-9906797, -16070310, 21134160, 12198166, -27064575, 708126, 387813, 13770293, -19134326, 10958663}, + FieldElement{22470984, 12369526, 23446014, -5441109, -21520802, -9698723, -11772496, -11574455, -25083830, 4271862}, + }, + { + FieldElement{-25169565, -10053642, -19909332, 15361595, -5984358, 2159192, 75375, -4278529, -32526221, 8469673}, + FieldElement{15854970, 4148314, -8893890, 7259002, 11666551, 13824734, -30531198, 2697372, 24154791, -9460943}, + FieldElement{15446137, -15806644, 29759747, 14019369, 30811221, -9610191, -31582008, 12840104, 24913809, 9815020}, + }, + { + FieldElement{-4709286, -5614269, -31841498, -12288893, -14443537, 10799414, -9103676, 13438769, 18735128, 9466238}, + FieldElement{11933045, 9281483, 5081055, -5183824, -2628162, -4905629, -7727821, -10896103, -22728655, 16199064}, + FieldElement{14576810, 379472, -26786533, -8317236, -29426508, -10812974, -102766, 1876699, 30801119, 2164795}, + }, + { + FieldElement{15995086, 3199873, 13672555, 13712240, -19378835, -4647646, -13081610, -15496269, -13492807, 1268052}, + FieldElement{-10290614, -3659039, -3286592, 10948818, 23037027, 3794475, -3470338, -12600221, -17055369, 3565904}, + FieldElement{29210088, -9419337, -5919792, -4952785, 10834811, -13327726, -16512102, -10820713, -27162222, -14030531}, + }, + { + FieldElement{-13161890, 15508588, 16663704, -8156150, -28349942, 9019123, -29183421, -3769423, 2244111, -14001979}, + FieldElement{-5152875, -3800936, -9306475, -6071583, 16243069, 14684434, -25673088, -16180800, 13491506, 4641841}, + FieldElement{10813417, 643330, -19188515, -728916, 30292062, -16600078, 27548447, -7721242, 14476989, -12767431}, + }, + { + FieldElement{10292079, 9984945, 6481436, 8279905, -7251514, 7032743, 27282937, -1644259, -27912810, 12651324}, + FieldElement{-31185513, -813383, 22271204, 11835308, 10201545, 15351028, 17099662, 3988035, 21721536, -3148940}, + FieldElement{10202177, -6545839, -31373232, -9574638, -32150642, -8119683, -12906320, 3852694, 13216206, 14842320}, + }, + { + FieldElement{-15815640, -10601066, -6538952, -7258995, -6984659, -6581778, -31500847, 13765824, -27434397, 9900184}, + FieldElement{14465505, -13833331, -32133984, -14738873, -27443187, 12990492, 33046193, 15796406, -7051866, -8040114}, + FieldElement{30924417, -8279620, 6359016, -12816335, 16508377, 9071735, -25488601, 15413635, 9524356, -7018878}, + }, + { + FieldElement{12274201, -13175547, 32627641, -1785326, 6736625, 13267305, 5237659, -5109483, 15663516, 4035784}, + FieldElement{-2951309, 8903985, 17349946, 601635, -16432815, -4612556, -13732739, -15889334, -22258478, 4659091}, + FieldElement{-16916263, -4952973, -30393711, -15158821, 20774812, 15897498, 5736189, 15026997, -2178256, -13455585}, + }, + }, + { + { + FieldElement{-8858980, -2219056, 28571666, -10155518, -474467, -10105698, -3801496, 278095, 23440562, -290208}, + FieldElement{10226241, -5928702, 15139956, 120818, -14867693, 5218603, 32937275, 11551483, -16571960, -7442864}, + FieldElement{17932739, -12437276, -24039557, 10749060, 11316803, 7535897, 22503767, 5561594, -3646624, 3898661}, + }, + { + FieldElement{7749907, -969567, -16339731, -16464, -25018111, 15122143, -1573531, 7152530, 21831162, 1245233}, + FieldElement{26958459, -14658026, 4314586, 8346991, -5677764, 11960072, -32589295, -620035, -30402091, -16716212}, + FieldElement{-12165896, 9166947, 33491384, 13673479, 29787085, 13096535, 6280834, 14587357, -22338025, 13987525}, + }, + { + FieldElement{-24349909, 7778775, 21116000, 15572597, -4833266, -5357778, -4300898, -5124639, -7469781, -2858068}, + FieldElement{9681908, -6737123, -31951644, 13591838, -6883821, 386950, 31622781, 6439245, -14581012, 4091397}, + FieldElement{-8426427, 1470727, -28109679, -1596990, 3978627, -5123623, -19622683, 12092163, 29077877, -14741988}, + }, + { + FieldElement{5269168, -6859726, -13230211, -8020715, 25932563, 1763552, -5606110, -5505881, -20017847, 2357889}, + FieldElement{32264008, -15407652, -5387735, -1160093, -2091322, -3946900, 23104804, -12869908, 5727338, 189038}, + FieldElement{14609123, -8954470, -6000566, -16622781, -14577387, -7743898, -26745169, 10942115, -25888931, -14884697}, + }, + { + FieldElement{20513500, 5557931, -15604613, 7829531, 26413943, -2019404, -21378968, 7471781, 13913677, -5137875}, + FieldElement{-25574376, 11967826, 29233242, 12948236, -6754465, 4713227, -8940970, 14059180, 12878652, 8511905}, + FieldElement{-25656801, 3393631, -2955415, -7075526, -2250709, 9366908, -30223418, 6812974, 5568676, -3127656}, + }, + { + FieldElement{11630004, 12144454, 2116339, 13606037, 27378885, 15676917, -17408753, -13504373, -14395196, 8070818}, + FieldElement{27117696, -10007378, -31282771, -5570088, 1127282, 12772488, -29845906, 10483306, -11552749, -1028714}, + FieldElement{10637467, -5688064, 5674781, 1072708, -26343588, -6982302, -1683975, 9177853, -27493162, 15431203}, + }, + { + FieldElement{20525145, 10892566, -12742472, 12779443, -29493034, 16150075, -28240519, 14943142, -15056790, -7935931}, + FieldElement{-30024462, 5626926, -551567, -9981087, 753598, 11981191, 25244767, -3239766, -3356550, 9594024}, + FieldElement{-23752644, 2636870, -5163910, -10103818, 585134, 7877383, 11345683, -6492290, 13352335, -10977084}, + }, + { + FieldElement{-1931799, -5407458, 3304649, -12884869, 17015806, -4877091, -29783850, -7752482, -13215537, -319204}, + FieldElement{20239939, 6607058, 6203985, 3483793, -18386976, -779229, -20723742, 15077870, -22750759, 14523817}, + FieldElement{27406042, -6041657, 27423596, -4497394, 4996214, 10002360, -28842031, -4545494, -30172742, -4805667}, + }, + }, + { + { + FieldElement{11374242, 12660715, 17861383, -12540833, 10935568, 1099227, -13886076, -9091740, -27727044, 11358504}, + FieldElement{-12730809, 10311867, 1510375, 10778093, -2119455, -9145702, 32676003, 11149336, -26123651, 4985768}, + FieldElement{-19096303, 341147, -6197485, -239033, 15756973, -8796662, -983043, 13794114, -19414307, -15621255}, + }, + { + FieldElement{6490081, 11940286, 25495923, -7726360, 8668373, -8751316, 3367603, 6970005, -1691065, -9004790}, + FieldElement{1656497, 13457317, 15370807, 6364910, 13605745, 8362338, -19174622, -5475723, -16796596, -5031438}, + FieldElement{-22273315, -13524424, -64685, -4334223, -18605636, -10921968, -20571065, -7007978, -99853, -10237333}, + }, + { + FieldElement{17747465, 10039260, 19368299, -4050591, -20630635, -16041286, 31992683, -15857976, -29260363, -5511971}, + FieldElement{31932027, -4986141, -19612382, 16366580, 22023614, 88450, 11371999, -3744247, 4882242, -10626905}, + FieldElement{29796507, 37186, 19818052, 10115756, -11829032, 3352736, 18551198, 3272828, -5190932, -4162409}, + }, + { + FieldElement{12501286, 4044383, -8612957, -13392385, -32430052, 5136599, -19230378, -3529697, 330070, -3659409}, + FieldElement{6384877, 2899513, 17807477, 7663917, -2358888, 12363165, 25366522, -8573892, -271295, 12071499}, + FieldElement{-8365515, -4042521, 25133448, -4517355, -6211027, 2265927, -32769618, 1936675, -5159697, 3829363}, + }, + { + FieldElement{28425966, -5835433, -577090, -4697198, -14217555, 6870930, 7921550, -6567787, 26333140, 14267664}, + FieldElement{-11067219, 11871231, 27385719, -10559544, -4585914, -11189312, 10004786, -8709488, -21761224, 8930324}, + FieldElement{-21197785, -16396035, 25654216, -1725397, 12282012, 11008919, 1541940, 4757911, -26491501, -16408940}, + }, + { + FieldElement{13537262, -7759490, -20604840, 10961927, -5922820, -13218065, -13156584, 6217254, -15943699, 13814990}, + FieldElement{-17422573, 15157790, 18705543, 29619, 24409717, -260476, 27361681, 9257833, -1956526, -1776914}, + FieldElement{-25045300, -10191966, 15366585, 15166509, -13105086, 8423556, -29171540, 12361135, -18685978, 4578290}, + }, + { + FieldElement{24579768, 3711570, 1342322, -11180126, -27005135, 14124956, -22544529, 14074919, 21964432, 8235257}, + FieldElement{-6528613, -2411497, 9442966, -5925588, 12025640, -1487420, -2981514, -1669206, 13006806, 2355433}, + FieldElement{-16304899, -13605259, -6632427, -5142349, 16974359, -10911083, 27202044, 1719366, 1141648, -12796236}, + }, + { + FieldElement{-12863944, -13219986, -8318266, -11018091, -6810145, -4843894, 13475066, -3133972, 32674895, 13715045}, + FieldElement{11423335, -5468059, 32344216, 8962751, 24989809, 9241752, -13265253, 16086212, -28740881, -15642093}, + FieldElement{-1409668, 12530728, -6368726, 10847387, 19531186, -14132160, -11709148, 7791794, -27245943, 4383347}, + }, + }, + { + { + FieldElement{-28970898, 5271447, -1266009, -9736989, -12455236, 16732599, -4862407, -4906449, 27193557, 6245191}, + FieldElement{-15193956, 5362278, -1783893, 2695834, 4960227, 12840725, 23061898, 3260492, 22510453, 8577507}, + FieldElement{-12632451, 11257346, -32692994, 13548177, -721004, 10879011, 31168030, 13952092, -29571492, -3635906}, + }, + { + FieldElement{3877321, -9572739, 32416692, 5405324, -11004407, -13656635, 3759769, 11935320, 5611860, 8164018}, + FieldElement{-16275802, 14667797, 15906460, 12155291, -22111149, -9039718, 32003002, -8832289, 5773085, -8422109}, + FieldElement{-23788118, -8254300, 1950875, 8937633, 18686727, 16459170, -905725, 12376320, 31632953, 190926}, + }, + { + FieldElement{-24593607, -16138885, -8423991, 13378746, 14162407, 6901328, -8288749, 4508564, -25341555, -3627528}, + FieldElement{8884438, -5884009, 6023974, 10104341, -6881569, -4941533, 18722941, -14786005, -1672488, 827625}, + FieldElement{-32720583, -16289296, -32503547, 7101210, 13354605, 2659080, -1800575, -14108036, -24878478, 1541286}, + }, + { + FieldElement{2901347, -1117687, 3880376, -10059388, -17620940, -3612781, -21802117, -3567481, 20456845, -1885033}, + FieldElement{27019610, 12299467, -13658288, -1603234, -12861660, -4861471, -19540150, -5016058, 29439641, 15138866}, + FieldElement{21536104, -6626420, -32447818, -10690208, -22408077, 5175814, -5420040, -16361163, 7779328, 109896}, + }, + { + FieldElement{30279744, 14648750, -8044871, 6425558, 13639621, -743509, 28698390, 12180118, 23177719, -554075}, + FieldElement{26572847, 3405927, -31701700, 12890905, -19265668, 5335866, -6493768, 2378492, 4439158, -13279347}, + FieldElement{-22716706, 3489070, -9225266, -332753, 18875722, -1140095, 14819434, -12731527, -17717757, -5461437}, + }, + { + FieldElement{-5056483, 16566551, 15953661, 3767752, -10436499, 15627060, -820954, 2177225, 8550082, -15114165}, + FieldElement{-18473302, 16596775, -381660, 15663611, 22860960, 15585581, -27844109, -3582739, -23260460, -8428588}, + FieldElement{-32480551, 15707275, -8205912, -5652081, 29464558, 2713815, -22725137, 15860482, -21902570, 1494193}, + }, + { + FieldElement{-19562091, -14087393, -25583872, -9299552, 13127842, 759709, 21923482, 16529112, 8742704, 12967017}, + FieldElement{-28464899, 1553205, 32536856, -10473729, -24691605, -406174, -8914625, -2933896, -29903758, 15553883}, + FieldElement{21877909, 3230008, 9881174, 10539357, -4797115, 2841332, 11543572, 14513274, 19375923, -12647961}, + }, + { + FieldElement{8832269, -14495485, 13253511, 5137575, 5037871, 4078777, 24880818, -6222716, 2862653, 9455043}, + FieldElement{29306751, 5123106, 20245049, -14149889, 9592566, 8447059, -2077124, -2990080, 15511449, 4789663}, + FieldElement{-20679756, 7004547, 8824831, -9434977, -4045704, -3750736, -5754762, 108893, 23513200, 16652362}, + }, + }, + { + { + FieldElement{-33256173, 4144782, -4476029, -6579123, 10770039, -7155542, -6650416, -12936300, -18319198, 10212860}, + FieldElement{2756081, 8598110, 7383731, -6859892, 22312759, -1105012, 21179801, 2600940, -9988298, -12506466}, + FieldElement{-24645692, 13317462, -30449259, -15653928, 21365574, -10869657, 11344424, 864440, -2499677, -16710063}, + }, + { + FieldElement{-26432803, 6148329, -17184412, -14474154, 18782929, -275997, -22561534, 211300, 2719757, 4940997}, + FieldElement{-1323882, 3911313, -6948744, 14759765, -30027150, 7851207, 21690126, 8518463, 26699843, 5276295}, + FieldElement{-13149873, -6429067, 9396249, 365013, 24703301, -10488939, 1321586, 149635, -15452774, 7159369}, + }, + { + FieldElement{9987780, -3404759, 17507962, 9505530, 9731535, -2165514, 22356009, 8312176, 22477218, -8403385}, + FieldElement{18155857, -16504990, 19744716, 9006923, 15154154, -10538976, 24256460, -4864995, -22548173, 9334109}, + FieldElement{2986088, -4911893, 10776628, -3473844, 10620590, -7083203, -21413845, 14253545, -22587149, 536906}, + }, + { + FieldElement{4377756, 8115836, 24567078, 15495314, 11625074, 13064599, 7390551, 10589625, 10838060, -15420424}, + FieldElement{-19342404, 867880, 9277171, -3218459, -14431572, -1986443, 19295826, -15796950, 6378260, 699185}, + FieldElement{7895026, 4057113, -7081772, -13077756, -17886831, -323126, -716039, 15693155, -5045064, -13373962}, + }, + { + FieldElement{-7737563, -5869402, -14566319, -7406919, 11385654, 13201616, 31730678, -10962840, -3918636, -9669325}, + FieldElement{10188286, -15770834, -7336361, 13427543, 22223443, 14896287, 30743455, 7116568, -21786507, 5427593}, + FieldElement{696102, 13206899, 27047647, -10632082, 15285305, -9853179, 10798490, -4578720, 19236243, 12477404}, + }, + { + FieldElement{-11229439, 11243796, -17054270, -8040865, -788228, -8167967, -3897669, 11180504, -23169516, 7733644}, + FieldElement{17800790, -14036179, -27000429, -11766671, 23887827, 3149671, 23466177, -10538171, 10322027, 15313801}, + FieldElement{26246234, 11968874, 32263343, -5468728, 6830755, -13323031, -15794704, -101982, -24449242, 10890804}, + }, + { + FieldElement{-31365647, 10271363, -12660625, -6267268, 16690207, -13062544, -14982212, 16484931, 25180797, -5334884}, + FieldElement{-586574, 10376444, -32586414, -11286356, 19801893, 10997610, 2276632, 9482883, 316878, 13820577}, + FieldElement{-9882808, -4510367, -2115506, 16457136, -11100081, 11674996, 30756178, -7515054, 30696930, -3712849}, + }, + { + FieldElement{32988917, -9603412, 12499366, 7910787, -10617257, -11931514, -7342816, -9985397, -32349517, 7392473}, + FieldElement{-8855661, 15927861, 9866406, -3649411, -2396914, -16655781, -30409476, -9134995, 25112947, -2926644}, + FieldElement{-2504044, -436966, 25621774, -5678772, 15085042, -5479877, -24884878, -13526194, 5537438, -13914319}, + }, + }, + { + { + FieldElement{-11225584, 2320285, -9584280, 10149187, -33444663, 5808648, -14876251, -1729667, 31234590, 6090599}, + FieldElement{-9633316, 116426, 26083934, 2897444, -6364437, -2688086, 609721, 15878753, -6970405, -9034768}, + FieldElement{-27757857, 247744, -15194774, -9002551, 23288161, -10011936, -23869595, 6503646, 20650474, 1804084}, + }, + { + FieldElement{-27589786, 15456424, 8972517, 8469608, 15640622, 4439847, 3121995, -10329713, 27842616, -202328}, + FieldElement{-15306973, 2839644, 22530074, 10026331, 4602058, 5048462, 28248656, 5031932, -11375082, 12714369}, + FieldElement{20807691, -7270825, 29286141, 11421711, -27876523, -13868230, -21227475, 1035546, -19733229, 12796920}, + }, + { + FieldElement{12076899, -14301286, -8785001, -11848922, -25012791, 16400684, -17591495, -12899438, 3480665, -15182815}, + FieldElement{-32361549, 5457597, 28548107, 7833186, 7303070, -11953545, -24363064, -15921875, -33374054, 2771025}, + FieldElement{-21389266, 421932, 26597266, 6860826, 22486084, -6737172, -17137485, -4210226, -24552282, 15673397}, + }, + { + FieldElement{-20184622, 2338216, 19788685, -9620956, -4001265, -8740893, -20271184, 4733254, 3727144, -12934448}, + FieldElement{6120119, 814863, -11794402, -622716, 6812205, -15747771, 2019594, 7975683, 31123697, -10958981}, + FieldElement{30069250, -11435332, 30434654, 2958439, 18399564, -976289, 12296869, 9204260, -16432438, 9648165}, + }, + { + FieldElement{32705432, -1550977, 30705658, 7451065, -11805606, 9631813, 3305266, 5248604, -26008332, -11377501}, + FieldElement{17219865, 2375039, -31570947, -5575615, -19459679, 9219903, 294711, 15298639, 2662509, -16297073}, + FieldElement{-1172927, -7558695, -4366770, -4287744, -21346413, -8434326, 32087529, -1222777, 32247248, -14389861}, + }, + { + FieldElement{14312628, 1221556, 17395390, -8700143, -4945741, -8684635, -28197744, -9637817, -16027623, -13378845}, + FieldElement{-1428825, -9678990, -9235681, 6549687, -7383069, -468664, 23046502, 9803137, 17597934, 2346211}, + FieldElement{18510800, 15337574, 26171504, 981392, -22241552, 7827556, -23491134, -11323352, 3059833, -11782870}, + }, + { + FieldElement{10141598, 6082907, 17829293, -1947643, 9830092, 13613136, -25556636, -5544586, -33502212, 3592096}, + FieldElement{33114168, -15889352, -26525686, -13343397, 33076705, 8716171, 1151462, 1521897, -982665, -6837803}, + FieldElement{-32939165, -4255815, 23947181, -324178, -33072974, -12305637, -16637686, 3891704, 26353178, 693168}, + }, + { + FieldElement{30374239, 1595580, -16884039, 13186931, 4600344, 406904, 9585294, -400668, 31375464, 14369965}, + FieldElement{-14370654, -7772529, 1510301, 6434173, -18784789, -6262728, 32732230, -13108839, 17901441, 16011505}, + FieldElement{18171223, -11934626, -12500402, 15197122, -11038147, -15230035, -19172240, -16046376, 8764035, 12309598}, + }, + }, + { + { + FieldElement{5975908, -5243188, -19459362, -9681747, -11541277, 14015782, -23665757, 1228319, 17544096, -10593782}, + FieldElement{5811932, -1715293, 3442887, -2269310, -18367348, -8359541, -18044043, -15410127, -5565381, 12348900}, + FieldElement{-31399660, 11407555, 25755363, 6891399, -3256938, 14872274, -24849353, 8141295, -10632534, -585479}, + }, + { + FieldElement{-12675304, 694026, -5076145, 13300344, 14015258, -14451394, -9698672, -11329050, 30944593, 1130208}, + FieldElement{8247766, -6710942, -26562381, -7709309, -14401939, -14648910, 4652152, 2488540, 23550156, -271232}, + FieldElement{17294316, -3788438, 7026748, 15626851, 22990044, 113481, 2267737, -5908146, -408818, -137719}, + }, + { + FieldElement{16091085, -16253926, 18599252, 7340678, 2137637, -1221657, -3364161, 14550936, 3260525, -7166271}, + FieldElement{-4910104, -13332887, 18550887, 10864893, -16459325, -7291596, -23028869, -13204905, -12748722, 2701326}, + FieldElement{-8574695, 16099415, 4629974, -16340524, -20786213, -6005432, -10018363, 9276971, 11329923, 1862132}, + }, + { + FieldElement{14763076, -15903608, -30918270, 3689867, 3511892, 10313526, -21951088, 12219231, -9037963, -940300}, + FieldElement{8894987, -3446094, 6150753, 3013931, 301220, 15693451, -31981216, -2909717, -15438168, 11595570}, + FieldElement{15214962, 3537601, -26238722, -14058872, 4418657, -15230761, 13947276, 10730794, -13489462, -4363670}, + }, + { + FieldElement{-2538306, 7682793, 32759013, 263109, -29984731, -7955452, -22332124, -10188635, 977108, 699994}, + FieldElement{-12466472, 4195084, -9211532, 550904, -15565337, 12917920, 19118110, -439841, -30534533, -14337913}, + FieldElement{31788461, -14507657, 4799989, 7372237, 8808585, -14747943, 9408237, -10051775, 12493932, -5409317}, + }, + { + FieldElement{-25680606, 5260744, -19235809, -6284470, -3695942, 16566087, 27218280, 2607121, 29375955, 6024730}, + FieldElement{842132, -2794693, -4763381, -8722815, 26332018, -12405641, 11831880, 6985184, -9940361, 2854096}, + FieldElement{-4847262, -7969331, 2516242, -5847713, 9695691, -7221186, 16512645, 960770, 12121869, 16648078}, + }, + { + FieldElement{-15218652, 14667096, -13336229, 2013717, 30598287, -464137, -31504922, -7882064, 20237806, 2838411}, + FieldElement{-19288047, 4453152, 15298546, -16178388, 22115043, -15972604, 12544294, -13470457, 1068881, -12499905}, + FieldElement{-9558883, -16518835, 33238498, 13506958, 30505848, -1114596, -8486907, -2630053, 12521378, 4845654}, + }, + { + FieldElement{-28198521, 10744108, -2958380, 10199664, 7759311, -13088600, 3409348, -873400, -6482306, -12885870}, + FieldElement{-23561822, 6230156, -20382013, 10655314, -24040585, -11621172, 10477734, -1240216, -3113227, 13974498}, + FieldElement{12966261, 15550616, -32038948, -1615346, 21025980, -629444, 5642325, 7188737, 18895762, 12629579}, + }, + }, + { + { + FieldElement{14741879, -14946887, 22177208, -11721237, 1279741, 8058600, 11758140, 789443, 32195181, 3895677}, + FieldElement{10758205, 15755439, -4509950, 9243698, -4879422, 6879879, -2204575, -3566119, -8982069, 4429647}, + FieldElement{-2453894, 15725973, -20436342, -10410672, -5803908, -11040220, -7135870, -11642895, 18047436, -15281743}, + }, + { + FieldElement{-25173001, -11307165, 29759956, 11776784, -22262383, -15820455, 10993114, -12850837, -17620701, -9408468}, + FieldElement{21987233, 700364, -24505048, 14972008, -7774265, -5718395, 32155026, 2581431, -29958985, 8773375}, + FieldElement{-25568350, 454463, -13211935, 16126715, 25240068, 8594567, 20656846, 12017935, -7874389, -13920155}, + }, + { + FieldElement{6028182, 6263078, -31011806, -11301710, -818919, 2461772, -31841174, -5468042, -1721788, -2776725}, + FieldElement{-12278994, 16624277, 987579, -5922598, 32908203, 1248608, 7719845, -4166698, 28408820, 6816612}, + FieldElement{-10358094, -8237829, 19549651, -12169222, 22082623, 16147817, 20613181, 13982702, -10339570, 5067943}, + }, + { + FieldElement{-30505967, -3821767, 12074681, 13582412, -19877972, 2443951, -19719286, 12746132, 5331210, -10105944}, + FieldElement{30528811, 3601899, -1957090, 4619785, -27361822, -15436388, 24180793, -12570394, 27679908, -1648928}, + FieldElement{9402404, -13957065, 32834043, 10838634, -26580150, -13237195, 26653274, -8685565, 22611444, -12715406}, + }, + { + FieldElement{22190590, 1118029, 22736441, 15130463, -30460692, -5991321, 19189625, -4648942, 4854859, 6622139}, + FieldElement{-8310738, -2953450, -8262579, -3388049, -10401731, -271929, 13424426, -3567227, 26404409, 13001963}, + FieldElement{-31241838, -15415700, -2994250, 8939346, 11562230, -12840670, -26064365, -11621720, -15405155, 11020693}, + }, + { + FieldElement{1866042, -7949489, -7898649, -10301010, 12483315, 13477547, 3175636, -12424163, 28761762, 1406734}, + FieldElement{-448555, -1777666, 13018551, 3194501, -9580420, -11161737, 24760585, -4347088, 25577411, -13378680}, + FieldElement{-24290378, 4759345, -690653, -1852816, 2066747, 10693769, -29595790, 9884936, -9368926, 4745410}, + }, + { + FieldElement{-9141284, 6049714, -19531061, -4341411, -31260798, 9944276, -15462008, -11311852, 10931924, -11931931}, + FieldElement{-16561513, 14112680, -8012645, 4817318, -8040464, -11414606, -22853429, 10856641, -20470770, 13434654}, + FieldElement{22759489, -10073434, -16766264, -1871422, 13637442, -10168091, 1765144, -12654326, 28445307, -5364710}, + }, + { + FieldElement{29875063, 12493613, 2795536, -3786330, 1710620, 15181182, -10195717, -8788675, 9074234, 1167180}, + FieldElement{-26205683, 11014233, -9842651, -2635485, -26908120, 7532294, -18716888, -9535498, 3843903, 9367684}, + FieldElement{-10969595, -6403711, 9591134, 9582310, 11349256, 108879, 16235123, 8601684, -139197, 4242895}, + }, + }, + { + { + FieldElement{22092954, -13191123, -2042793, -11968512, 32186753, -11517388, -6574341, 2470660, -27417366, 16625501}, + FieldElement{-11057722, 3042016, 13770083, -9257922, 584236, -544855, -7770857, 2602725, -27351616, 14247413}, + FieldElement{6314175, -10264892, -32772502, 15957557, -10157730, 168750, -8618807, 14290061, 27108877, -1180880}, + }, + { + FieldElement{-8586597, -7170966, 13241782, 10960156, -32991015, -13794596, 33547976, -11058889, -27148451, 981874}, + FieldElement{22833440, 9293594, -32649448, -13618667, -9136966, 14756819, -22928859, -13970780, -10479804, -16197962}, + FieldElement{-7768587, 3326786, -28111797, 10783824, 19178761, 14905060, 22680049, 13906969, -15933690, 3797899}, + }, + { + FieldElement{21721356, -4212746, -12206123, 9310182, -3882239, -13653110, 23740224, -2709232, 20491983, -8042152}, + FieldElement{9209270, -15135055, -13256557, -6167798, -731016, 15289673, 25947805, 15286587, 30997318, -6703063}, + FieldElement{7392032, 16618386, 23946583, -8039892, -13265164, -1533858, -14197445, -2321576, 17649998, -250080}, + }, + { + FieldElement{-9301088, -14193827, 30609526, -3049543, -25175069, -1283752, -15241566, -9525724, -2233253, 7662146}, + FieldElement{-17558673, 1763594, -33114336, 15908610, -30040870, -12174295, 7335080, -8472199, -3174674, 3440183}, + FieldElement{-19889700, -5977008, -24111293, -9688870, 10799743, -16571957, 40450, -4431835, 4862400, 1133}, + }, + { + FieldElement{-32856209, -7873957, -5422389, 14860950, -16319031, 7956142, 7258061, 311861, -30594991, -7379421}, + FieldElement{-3773428, -1565936, 28985340, 7499440, 24445838, 9325937, 29727763, 16527196, 18278453, 15405622}, + FieldElement{-4381906, 8508652, -19898366, -3674424, -5984453, 15149970, -13313598, 843523, -21875062, 13626197}, + }, + { + FieldElement{2281448, -13487055, -10915418, -2609910, 1879358, 16164207, -10783882, 3953792, 13340839, 15928663}, + FieldElement{31727126, -7179855, -18437503, -8283652, 2875793, -16390330, -25269894, -7014826, -23452306, 5964753}, + FieldElement{4100420, -5959452, -17179337, 6017714, -18705837, 12227141, -26684835, 11344144, 2538215, -7570755}, + }, + { + FieldElement{-9433605, 6123113, 11159803, -2156608, 30016280, 14966241, -20474983, 1485421, -629256, -15958862}, + FieldElement{-26804558, 4260919, 11851389, 9658551, -32017107, 16367492, -20205425, -13191288, 11659922, -11115118}, + FieldElement{26180396, 10015009, -30844224, -8581293, 5418197, 9480663, 2231568, -10170080, 33100372, -1306171}, + }, + { + FieldElement{15121113, -5201871, -10389905, 15427821, -27509937, -15992507, 21670947, 4486675, -5931810, -14466380}, + FieldElement{16166486, -9483733, -11104130, 6023908, -31926798, -1364923, 2340060, -16254968, -10735770, -10039824}, + FieldElement{28042865, -3557089, -12126526, 12259706, -3717498, -6945899, 6766453, -8689599, 18036436, 5803270}, + }, + }, + { + { + FieldElement{-817581, 6763912, 11803561, 1585585, 10958447, -2671165, 23855391, 4598332, -6159431, -14117438}, + FieldElement{-31031306, -14256194, 17332029, -2383520, 31312682, -5967183, 696309, 50292, -20095739, 11763584}, + FieldElement{-594563, -2514283, -32234153, 12643980, 12650761, 14811489, 665117, -12613632, -19773211, -10713562}, + }, + { + FieldElement{30464590, -11262872, -4127476, -12734478, 19835327, -7105613, -24396175, 2075773, -17020157, 992471}, + FieldElement{18357185, -6994433, 7766382, 16342475, -29324918, 411174, 14578841, 8080033, -11574335, -10601610}, + FieldElement{19598397, 10334610, 12555054, 2555664, 18821899, -10339780, 21873263, 16014234, 26224780, 16452269}, + }, + { + FieldElement{-30223925, 5145196, 5944548, 16385966, 3976735, 2009897, -11377804, -7618186, -20533829, 3698650}, + FieldElement{14187449, 3448569, -10636236, -10810935, -22663880, -3433596, 7268410, -10890444, 27394301, 12015369}, + FieldElement{19695761, 16087646, 28032085, 12999827, 6817792, 11427614, 20244189, -1312777, -13259127, -3402461}, + }, + { + FieldElement{30860103, 12735208, -1888245, -4699734, -16974906, 2256940, -8166013, 12298312, -8550524, -10393462}, + FieldElement{-5719826, -11245325, -1910649, 15569035, 26642876, -7587760, -5789354, -15118654, -4976164, 12651793}, + FieldElement{-2848395, 9953421, 11531313, -5282879, 26895123, -12697089, -13118820, -16517902, 9768698, -2533218}, + }, + { + FieldElement{-24719459, 1894651, -287698, -4704085, 15348719, -8156530, 32767513, 12765450, 4940095, 10678226}, + FieldElement{18860224, 15980149, -18987240, -1562570, -26233012, -11071856, -7843882, 13944024, -24372348, 16582019}, + FieldElement{-15504260, 4970268, -29893044, 4175593, -20993212, -2199756, -11704054, 15444560, -11003761, 7989037}, + }, + { + FieldElement{31490452, 5568061, -2412803, 2182383, -32336847, 4531686, -32078269, 6200206, -19686113, -14800171}, + FieldElement{-17308668, -15879940, -31522777, -2831, -32887382, 16375549, 8680158, -16371713, 28550068, -6857132}, + FieldElement{-28126887, -5688091, 16837845, -1820458, -6850681, 12700016, -30039981, 4364038, 1155602, 5988841}, + }, + { + FieldElement{21890435, -13272907, -12624011, 12154349, -7831873, 15300496, 23148983, -4470481, 24618407, 8283181}, + FieldElement{-33136107, -10512751, 9975416, 6841041, -31559793, 16356536, 3070187, -7025928, 1466169, 10740210}, + FieldElement{-1509399, -15488185, -13503385, -10655916, 32799044, 909394, -13938903, -5779719, -32164649, -15327040}, + }, + { + FieldElement{3960823, -14267803, -28026090, -15918051, -19404858, 13146868, 15567327, 951507, -3260321, -573935}, + FieldElement{24740841, 5052253, -30094131, 8961361, 25877428, 6165135, -24368180, 14397372, -7380369, -6144105}, + FieldElement{-28888365, 3510803, -28103278, -1158478, -11238128, -10631454, -15441463, -14453128, -1625486, -6494814}, + }, + }, + { + { + FieldElement{793299, -9230478, 8836302, -6235707, -27360908, -2369593, 33152843, -4885251, -9906200, -621852}, + FieldElement{5666233, 525582, 20782575, -8038419, -24538499, 14657740, 16099374, 1468826, -6171428, -15186581}, + FieldElement{-4859255, -3779343, -2917758, -6748019, 7778750, 11688288, -30404353, -9871238, -1558923, -9863646}, + }, + { + FieldElement{10896332, -7719704, 824275, 472601, -19460308, 3009587, 25248958, 14783338, -30581476, -15757844}, + FieldElement{10566929, 12612572, -31944212, 11118703, -12633376, 12362879, 21752402, 8822496, 24003793, 14264025}, + FieldElement{27713862, -7355973, -11008240, 9227530, 27050101, 2504721, 23886875, -13117525, 13958495, -5732453}, + }, + { + FieldElement{-23481610, 4867226, -27247128, 3900521, 29838369, -8212291, -31889399, -10041781, 7340521, -15410068}, + FieldElement{4646514, -8011124, -22766023, -11532654, 23184553, 8566613, 31366726, -1381061, -15066784, -10375192}, + FieldElement{-17270517, 12723032, -16993061, 14878794, 21619651, -6197576, 27584817, 3093888, -8843694, 3849921}, + }, + { + FieldElement{-9064912, 2103172, 25561640, -15125738, -5239824, 9582958, 32477045, -9017955, 5002294, -15550259}, + FieldElement{-12057553, -11177906, 21115585, -13365155, 8808712, -12030708, 16489530, 13378448, -25845716, 12741426}, + FieldElement{-5946367, 10645103, -30911586, 15390284, -3286982, -7118677, 24306472, 15852464, 28834118, -7646072}, + }, + { + FieldElement{-17335748, -9107057, -24531279, 9434953, -8472084, -583362, -13090771, 455841, 20461858, 5491305}, + FieldElement{13669248, -16095482, -12481974, -10203039, -14569770, -11893198, -24995986, 11293807, -28588204, -9421832}, + FieldElement{28497928, 6272777, -33022994, 14470570, 8906179, -1225630, 18504674, -14165166, 29867745, -8795943}, + }, + { + FieldElement{-16207023, 13517196, -27799630, -13697798, 24009064, -6373891, -6367600, -13175392, 22853429, -4012011}, + FieldElement{24191378, 16712145, -13931797, 15217831, 14542237, 1646131, 18603514, -11037887, 12876623, -2112447}, + FieldElement{17902668, 4518229, -411702, -2829247, 26878217, 5258055, -12860753, 608397, 16031844, 3723494}, + }, + { + FieldElement{-28632773, 12763728, -20446446, 7577504, 33001348, -13017745, 17558842, -7872890, 23896954, -4314245}, + FieldElement{-20005381, -12011952, 31520464, 605201, 2543521, 5991821, -2945064, 7229064, -9919646, -8826859}, + FieldElement{28816045, 298879, -28165016, -15920938, 19000928, -1665890, -12680833, -2949325, -18051778, -2082915}, + }, + { + FieldElement{16000882, -344896, 3493092, -11447198, -29504595, -13159789, 12577740, 16041268, -19715240, 7847707}, + FieldElement{10151868, 10572098, 27312476, 7922682, 14825339, 4723128, -32855931, -6519018, -10020567, 3852848}, + FieldElement{-11430470, 15697596, -21121557, -4420647, 5386314, 15063598, 16514493, -15932110, 29330899, -15076224}, + }, + }, + { + { + FieldElement{-25499735, -4378794, -15222908, -6901211, 16615731, 2051784, 3303702, 15490, -27548796, 12314391}, + FieldElement{15683520, -6003043, 18109120, -9980648, 15337968, -5997823, -16717435, 15921866, 16103996, -3731215}, + FieldElement{-23169824, -10781249, 13588192, -1628807, -3798557, -1074929, -19273607, 5402699, -29815713, -9841101}, + }, + { + FieldElement{23190676, 2384583, -32714340, 3462154, -29903655, -1529132, -11266856, 8911517, -25205859, 2739713}, + FieldElement{21374101, -3554250, -33524649, 9874411, 15377179, 11831242, -33529904, 6134907, 4931255, 11987849}, + FieldElement{-7732, -2978858, -16223486, 7277597, 105524, -322051, -31480539, 13861388, -30076310, 10117930}, + }, + { + FieldElement{-29501170, -10744872, -26163768, 13051539, -25625564, 5089643, -6325503, 6704079, 12890019, 15728940}, + FieldElement{-21972360, -11771379, -951059, -4418840, 14704840, 2695116, 903376, -10428139, 12885167, 8311031}, + FieldElement{-17516482, 5352194, 10384213, -13811658, 7506451, 13453191, 26423267, 4384730, 1888765, -5435404}, + }, + { + FieldElement{-25817338, -3107312, -13494599, -3182506, 30896459, -13921729, -32251644, -12707869, -19464434, -3340243}, + FieldElement{-23607977, -2665774, -526091, 4651136, 5765089, 4618330, 6092245, 14845197, 17151279, -9854116}, + FieldElement{-24830458, -12733720, -15165978, 10367250, -29530908, -265356, 22825805, -7087279, -16866484, 16176525}, + }, + { + FieldElement{-23583256, 6564961, 20063689, 3798228, -4740178, 7359225, 2006182, -10363426, -28746253, -10197509}, + FieldElement{-10626600, -4486402, -13320562, -5125317, 3432136, -6393229, 23632037, -1940610, 32808310, 1099883}, + FieldElement{15030977, 5768825, -27451236, -2887299, -6427378, -15361371, -15277896, -6809350, 2051441, -15225865}, + }, + { + FieldElement{-3362323, -7239372, 7517890, 9824992, 23555850, 295369, 5148398, -14154188, -22686354, 16633660}, + FieldElement{4577086, -16752288, 13249841, -15304328, 19958763, -14537274, 18559670, -10759549, 8402478, -9864273}, + FieldElement{-28406330, -1051581, -26790155, -907698, -17212414, -11030789, 9453451, -14980072, 17983010, 9967138}, + }, + { + FieldElement{-25762494, 6524722, 26585488, 9969270, 24709298, 1220360, -1677990, 7806337, 17507396, 3651560}, + FieldElement{-10420457, -4118111, 14584639, 15971087, -15768321, 8861010, 26556809, -5574557, -18553322, -11357135}, + FieldElement{2839101, 14284142, 4029895, 3472686, 14402957, 12689363, -26642121, 8459447, -5605463, -7621941}, + }, + { + FieldElement{-4839289, -3535444, 9744961, 2871048, 25113978, 3187018, -25110813, -849066, 17258084, -7977739}, + FieldElement{18164541, -10595176, -17154882, -1542417, 19237078, -9745295, 23357533, -15217008, 26908270, 12150756}, + FieldElement{-30264870, -7647865, 5112249, -7036672, -1499807, -6974257, 43168, -5537701, -32302074, 16215819}, + }, + }, + { + { + FieldElement{-6898905, 9824394, -12304779, -4401089, -31397141, -6276835, 32574489, 12532905, -7503072, -8675347}, + FieldElement{-27343522, -16515468, -27151524, -10722951, 946346, 16291093, 254968, 7168080, 21676107, -1943028}, + FieldElement{21260961, -8424752, -16831886, -11920822, -23677961, 3968121, -3651949, -6215466, -3556191, -7913075}, + }, + { + FieldElement{16544754, 13250366, -16804428, 15546242, -4583003, 12757258, -2462308, -8680336, -18907032, -9662799}, + FieldElement{-2415239, -15577728, 18312303, 4964443, -15272530, -12653564, 26820651, 16690659, 25459437, -4564609}, + FieldElement{-25144690, 11425020, 28423002, -11020557, -6144921, -15826224, 9142795, -2391602, -6432418, -1644817}, + }, + { + FieldElement{-23104652, 6253476, 16964147, -3768872, -25113972, -12296437, -27457225, -16344658, 6335692, 7249989}, + FieldElement{-30333227, 13979675, 7503222, -12368314, -11956721, -4621693, -30272269, 2682242, 25993170, -12478523}, + FieldElement{4364628, 5930691, 32304656, -10044554, -8054781, 15091131, 22857016, -10598955, 31820368, 15075278}, + }, + { + FieldElement{31879134, -8918693, 17258761, 90626, -8041836, -4917709, 24162788, -9650886, -17970238, 12833045}, + FieldElement{19073683, 14851414, -24403169, -11860168, 7625278, 11091125, -19619190, 2074449, -9413939, 14905377}, + FieldElement{24483667, -11935567, -2518866, -11547418, -1553130, 15355506, -25282080, 9253129, 27628530, -7555480}, + }, + { + FieldElement{17597607, 8340603, 19355617, 552187, 26198470, -3176583, 4593324, -9157582, -14110875, 15297016}, + FieldElement{510886, 14337390, -31785257, 16638632, 6328095, 2713355, -20217417, -11864220, 8683221, 2921426}, + FieldElement{18606791, 11874196, 27155355, -5281482, -24031742, 6265446, -25178240, -1278924, 4674690, 13890525}, + }, + { + FieldElement{13609624, 13069022, -27372361, -13055908, 24360586, 9592974, 14977157, 9835105, 4389687, 288396}, + FieldElement{9922506, -519394, 13613107, 5883594, -18758345, -434263, -12304062, 8317628, 23388070, 16052080}, + FieldElement{12720016, 11937594, -31970060, -5028689, 26900120, 8561328, -20155687, -11632979, -14754271, -10812892}, + }, + { + FieldElement{15961858, 14150409, 26716931, -665832, -22794328, 13603569, 11829573, 7467844, -28822128, 929275}, + FieldElement{11038231, -11582396, -27310482, -7316562, -10498527, -16307831, -23479533, -9371869, -21393143, 2465074}, + FieldElement{20017163, -4323226, 27915242, 1529148, 12396362, 15675764, 13817261, -9658066, 2463391, -4622140}, + }, + { + FieldElement{-16358878, -12663911, -12065183, 4996454, -1256422, 1073572, 9583558, 12851107, 4003896, 12673717}, + FieldElement{-1731589, -15155870, -3262930, 16143082, 19294135, 13385325, 14741514, -9103726, 7903886, 2348101}, + FieldElement{24536016, -16515207, 12715592, -3862155, 1511293, 10047386, -3842346, -7129159, -28377538, 10048127}, + }, + }, + { + { + FieldElement{-12622226, -6204820, 30718825, 2591312, -10617028, 12192840, 18873298, -7297090, -32297756, 15221632}, + FieldElement{-26478122, -11103864, 11546244, -1852483, 9180880, 7656409, -21343950, 2095755, 29769758, 6593415}, + FieldElement{-31994208, -2907461, 4176912, 3264766, 12538965, -868111, 26312345, -6118678, 30958054, 8292160}, + }, + { + FieldElement{31429822, -13959116, 29173532, 15632448, 12174511, -2760094, 32808831, 3977186, 26143136, -3148876}, + FieldElement{22648901, 1402143, -22799984, 13746059, 7936347, 365344, -8668633, -1674433, -3758243, -2304625}, + FieldElement{-15491917, 8012313, -2514730, -12702462, -23965846, -10254029, -1612713, -1535569, -16664475, 8194478}, + }, + { + FieldElement{27338066, -7507420, -7414224, 10140405, -19026427, -6589889, 27277191, 8855376, 28572286, 3005164}, + FieldElement{26287124, 4821776, 25476601, -4145903, -3764513, -15788984, -18008582, 1182479, -26094821, -13079595}, + FieldElement{-7171154, 3178080, 23970071, 6201893, -17195577, -4489192, -21876275, -13982627, 32208683, -1198248}, + }, + { + FieldElement{-16657702, 2817643, -10286362, 14811298, 6024667, 13349505, -27315504, -10497842, -27672585, -11539858}, + FieldElement{15941029, -9405932, -21367050, 8062055, 31876073, -238629, -15278393, -1444429, 15397331, -4130193}, + FieldElement{8934485, -13485467, -23286397, -13423241, -32446090, 14047986, 31170398, -1441021, -27505566, 15087184}, + }, + { + FieldElement{-18357243, -2156491, 24524913, -16677868, 15520427, -6360776, -15502406, 11461896, 16788528, -5868942}, + FieldElement{-1947386, 16013773, 21750665, 3714552, -17401782, -16055433, -3770287, -10323320, 31322514, -11615635}, + FieldElement{21426655, -5650218, -13648287, -5347537, -28812189, -4920970, -18275391, -14621414, 13040862, -12112948}, + }, + { + FieldElement{11293895, 12478086, -27136401, 15083750, -29307421, 14748872, 14555558, -13417103, 1613711, 4896935}, + FieldElement{-25894883, 15323294, -8489791, -8057900, 25967126, -13425460, 2825960, -4897045, -23971776, -11267415}, + FieldElement{-15924766, -5229880, -17443532, 6410664, 3622847, 10243618, 20615400, 12405433, -23753030, -8436416}, + }, + { + FieldElement{-7091295, 12556208, -20191352, 9025187, -17072479, 4333801, 4378436, 2432030, 23097949, -566018}, + FieldElement{4565804, -16025654, 20084412, -7842817, 1724999, 189254, 24767264, 10103221, -18512313, 2424778}, + FieldElement{366633, -11976806, 8173090, -6890119, 30788634, 5745705, -7168678, 1344109, -3642553, 12412659}, + }, + { + FieldElement{-24001791, 7690286, 14929416, -168257, -32210835, -13412986, 24162697, -15326504, -3141501, 11179385}, + FieldElement{18289522, -14724954, 8056945, 16430056, -21729724, 7842514, -6001441, -1486897, -18684645, -11443503}, + FieldElement{476239, 6601091, -6152790, -9723375, 17503545, -4863900, 27672959, 13403813, 11052904, 5219329}, + }, + }, + { + { + FieldElement{20678546, -8375738, -32671898, 8849123, -5009758, 14574752, 31186971, -3973730, 9014762, -8579056}, + FieldElement{-13644050, -10350239, -15962508, 5075808, -1514661, -11534600, -33102500, 9160280, 8473550, -3256838}, + FieldElement{24900749, 14435722, 17209120, -15292541, -22592275, 9878983, -7689309, -16335821, -24568481, 11788948}, + }, + { + FieldElement{-3118155, -11395194, -13802089, 14797441, 9652448, -6845904, -20037437, 10410733, -24568470, -1458691}, + FieldElement{-15659161, 16736706, -22467150, 10215878, -9097177, 7563911, 11871841, -12505194, -18513325, 8464118}, + FieldElement{-23400612, 8348507, -14585951, -861714, -3950205, -6373419, 14325289, 8628612, 33313881, -8370517}, + }, + { + FieldElement{-20186973, -4967935, 22367356, 5271547, -1097117, -4788838, -24805667, -10236854, -8940735, -5818269}, + FieldElement{-6948785, -1795212, -32625683, -16021179, 32635414, -7374245, 15989197, -12838188, 28358192, -4253904}, + FieldElement{-23561781, -2799059, -32351682, -1661963, -9147719, 10429267, -16637684, 4072016, -5351664, 5596589}, + }, + { + FieldElement{-28236598, -3390048, 12312896, 6213178, 3117142, 16078565, 29266239, 2557221, 1768301, 15373193}, + FieldElement{-7243358, -3246960, -4593467, -7553353, -127927, -912245, -1090902, -4504991, -24660491, 3442910}, + FieldElement{-30210571, 5124043, 14181784, 8197961, 18964734, -11939093, 22597931, 7176455, -18585478, 13365930}, + }, + { + FieldElement{-7877390, -1499958, 8324673, 4690079, 6261860, 890446, 24538107, -8570186, -9689599, -3031667}, + FieldElement{25008904, -10771599, -4305031, -9638010, 16265036, 15721635, 683793, -11823784, 15723479, -15163481}, + FieldElement{-9660625, 12374379, -27006999, -7026148, -7724114, -12314514, 11879682, 5400171, 519526, -1235876}, + }, + { + FieldElement{22258397, -16332233, -7869817, 14613016, -22520255, -2950923, -20353881, 7315967, 16648397, 7605640}, + FieldElement{-8081308, -8464597, -8223311, 9719710, 19259459, -15348212, 23994942, -5281555, -9468848, 4763278}, + FieldElement{-21699244, 9220969, -15730624, 1084137, -25476107, -2852390, 31088447, -7764523, -11356529, 728112}, + }, + { + FieldElement{26047220, -11751471, -6900323, -16521798, 24092068, 9158119, -4273545, -12555558, -29365436, -5498272}, + FieldElement{17510331, -322857, 5854289, 8403524, 17133918, -3112612, -28111007, 12327945, 10750447, 10014012}, + FieldElement{-10312768, 3936952, 9156313, -8897683, 16498692, -994647, -27481051, -666732, 3424691, 7540221}, + }, + { + FieldElement{30322361, -6964110, 11361005, -4143317, 7433304, 4989748, -7071422, -16317219, -9244265, 15258046}, + FieldElement{13054562, -2779497, 19155474, 469045, -12482797, 4566042, 5631406, 2711395, 1062915, -5136345}, + FieldElement{-19240248, -11254599, -29509029, -7499965, -5835763, 13005411, -6066489, 12194497, 32960380, 1459310}, + }, + }, + { + { + FieldElement{19852034, 7027924, 23669353, 10020366, 8586503, -6657907, 394197, -6101885, 18638003, -11174937}, + FieldElement{31395534, 15098109, 26581030, 8030562, -16527914, -5007134, 9012486, -7584354, -6643087, -5442636}, + FieldElement{-9192165, -2347377, -1997099, 4529534, 25766844, 607986, -13222, 9677543, -32294889, -6456008}, + }, + { + FieldElement{-2444496, -149937, 29348902, 8186665, 1873760, 12489863, -30934579, -7839692, -7852844, -8138429}, + FieldElement{-15236356, -15433509, 7766470, 746860, 26346930, -10221762, -27333451, 10754588, -9431476, 5203576}, + FieldElement{31834314, 14135496, -770007, 5159118, 20917671, -16768096, -7467973, -7337524, 31809243, 7347066}, + }, + { + FieldElement{-9606723, -11874240, 20414459, 13033986, 13716524, -11691881, 19797970, -12211255, 15192876, -2087490}, + FieldElement{-12663563, -2181719, 1168162, -3804809, 26747877, -14138091, 10609330, 12694420, 33473243, -13382104}, + FieldElement{33184999, 11180355, 15832085, -11385430, -1633671, 225884, 15089336, -11023903, -6135662, 14480053}, + }, + { + FieldElement{31308717, -5619998, 31030840, -1897099, 15674547, -6582883, 5496208, 13685227, 27595050, 8737275}, + FieldElement{-20318852, -15150239, 10933843, -16178022, 8335352, -7546022, -31008351, -12610604, 26498114, 66511}, + FieldElement{22644454, -8761729, -16671776, 4884562, -3105614, -13559366, 30540766, -4286747, -13327787, -7515095}, + }, + { + FieldElement{-28017847, 9834845, 18617207, -2681312, -3401956, -13307506, 8205540, 13585437, -17127465, 15115439}, + FieldElement{23711543, -672915, 31206561, -8362711, 6164647, -9709987, -33535882, -1426096, 8236921, 16492939}, + FieldElement{-23910559, -13515526, -26299483, -4503841, 25005590, -7687270, 19574902, 10071562, 6708380, -6222424}, + }, + { + FieldElement{2101391, -4930054, 19702731, 2367575, -15427167, 1047675, 5301017, 9328700, 29955601, -11678310}, + FieldElement{3096359, 9271816, -21620864, -15521844, -14847996, -7592937, -25892142, -12635595, -9917575, 6216608}, + FieldElement{-32615849, 338663, -25195611, 2510422, -29213566, -13820213, 24822830, -6146567, -26767480, 7525079}, + }, + { + FieldElement{-23066649, -13985623, 16133487, -7896178, -3389565, 778788, -910336, -2782495, -19386633, 11994101}, + FieldElement{21691500, -13624626, -641331, -14367021, 3285881, -3483596, -25064666, 9718258, -7477437, 13381418}, + FieldElement{18445390, -4202236, 14979846, 11622458, -1727110, -3582980, 23111648, -6375247, 28535282, 15779576}, + }, + { + FieldElement{30098053, 3089662, -9234387, 16662135, -21306940, 11308411, -14068454, 12021730, 9955285, -16303356}, + FieldElement{9734894, -14576830, -7473633, -9138735, 2060392, 11313496, -18426029, 9924399, 20194861, 13380996}, + FieldElement{-26378102, -7965207, -22167821, 15789297, -18055342, -6168792, -1984914, 15707771, 26342023, 10146099}, + }, + }, + { + { + FieldElement{-26016874, -219943, 21339191, -41388, 19745256, -2878700, -29637280, 2227040, 21612326, -545728}, + FieldElement{-13077387, 1184228, 23562814, -5970442, -20351244, -6348714, 25764461, 12243797, -20856566, 11649658}, + FieldElement{-10031494, 11262626, 27384172, 2271902, 26947504, -15997771, 39944, 6114064, 33514190, 2333242}, + }, + { + FieldElement{-21433588, -12421821, 8119782, 7219913, -21830522, -9016134, -6679750, -12670638, 24350578, -13450001}, + FieldElement{-4116307, -11271533, -23886186, 4843615, -30088339, 690623, -31536088, -10406836, 8317860, 12352766}, + FieldElement{18200138, -14475911, -33087759, -2696619, -23702521, -9102511, -23552096, -2287550, 20712163, 6719373}, + }, + { + FieldElement{26656208, 6075253, -7858556, 1886072, -28344043, 4262326, 11117530, -3763210, 26224235, -3297458}, + FieldElement{-17168938, -14854097, -3395676, -16369877, -19954045, 14050420, 21728352, 9493610, 18620611, -16428628}, + FieldElement{-13323321, 13325349, 11432106, 5964811, 18609221, 6062965, -5269471, -9725556, -30701573, -16479657}, + }, + { + FieldElement{-23860538, -11233159, 26961357, 1640861, -32413112, -16737940, 12248509, -5240639, 13735342, 1934062}, + FieldElement{25089769, 6742589, 17081145, -13406266, 21909293, -16067981, -15136294, -3765346, -21277997, 5473616}, + FieldElement{31883677, -7961101, 1083432, -11572403, 22828471, 13290673, -7125085, 12469656, 29111212, -5451014}, + }, + { + FieldElement{24244947, -15050407, -26262976, 2791540, -14997599, 16666678, 24367466, 6388839, -10295587, 452383}, + FieldElement{-25640782, -3417841, 5217916, 16224624, 19987036, -4082269, -24236251, -5915248, 15766062, 8407814}, + FieldElement{-20406999, 13990231, 15495425, 16395525, 5377168, 15166495, -8917023, -4388953, -8067909, 2276718}, + }, + { + FieldElement{30157918, 12924066, -17712050, 9245753, 19895028, 3368142, -23827587, 5096219, 22740376, -7303417}, + FieldElement{2041139, -14256350, 7783687, 13876377, -25946985, -13352459, 24051124, 13742383, -15637599, 13295222}, + FieldElement{33338237, -8505733, 12532113, 7977527, 9106186, -1715251, -17720195, -4612972, -4451357, -14669444}, + }, + { + FieldElement{-20045281, 5454097, -14346548, 6447146, 28862071, 1883651, -2469266, -4141880, 7770569, 9620597}, + FieldElement{23208068, 7979712, 33071466, 8149229, 1758231, -10834995, 30945528, -1694323, -33502340, -14767970}, + FieldElement{1439958, -16270480, -1079989, -793782, 4625402, 10647766, -5043801, 1220118, 30494170, -11440799}, + }, + { + FieldElement{-5037580, -13028295, -2970559, -3061767, 15640974, -6701666, -26739026, 926050, -1684339, -13333647}, + FieldElement{13908495, -3549272, 30919928, -6273825, -21521863, 7989039, 9021034, 9078865, 3353509, 4033511}, + FieldElement{-29663431, -15113610, 32259991, -344482, 24295849, -12912123, 23161163, 8839127, 27485041, 7356032}, + }, + }, + { + { + FieldElement{9661027, 705443, 11980065, -5370154, -1628543, 14661173, -6346142, 2625015, 28431036, -16771834}, + FieldElement{-23839233, -8311415, -25945511, 7480958, -17681669, -8354183, -22545972, 14150565, 15970762, 4099461}, + FieldElement{29262576, 16756590, 26350592, -8793563, 8529671, -11208050, 13617293, -9937143, 11465739, 8317062}, + }, + { + FieldElement{-25493081, -6962928, 32500200, -9419051, -23038724, -2302222, 14898637, 3848455, 20969334, -5157516}, + FieldElement{-20384450, -14347713, -18336405, 13884722, -33039454, 2842114, -21610826, -3649888, 11177095, 14989547}, + FieldElement{-24496721, -11716016, 16959896, 2278463, 12066309, 10137771, 13515641, 2581286, -28487508, 9930240}, + }, + { + FieldElement{-17751622, -2097826, 16544300, -13009300, -15914807, -14949081, 18345767, -13403753, 16291481, -5314038}, + FieldElement{-33229194, 2553288, 32678213, 9875984, 8534129, 6889387, -9676774, 6957617, 4368891, 9788741}, + FieldElement{16660756, 7281060, -10830758, 12911820, 20108584, -8101676, -21722536, -8613148, 16250552, -11111103}, + }, + { + FieldElement{-19765507, 2390526, -16551031, 14161980, 1905286, 6414907, 4689584, 10604807, -30190403, 4782747}, + FieldElement{-1354539, 14736941, -7367442, -13292886, 7710542, -14155590, -9981571, 4383045, 22546403, 437323}, + FieldElement{31665577, -12180464, -16186830, 1491339, -18368625, 3294682, 27343084, 2786261, -30633590, -14097016}, + }, + { + FieldElement{-14467279, -683715, -33374107, 7448552, 19294360, 14334329, -19690631, 2355319, -19284671, -6114373}, + FieldElement{15121312, -15796162, 6377020, -6031361, -10798111, -12957845, 18952177, 15496498, -29380133, 11754228}, + FieldElement{-2637277, -13483075, 8488727, -14303896, 12728761, -1622493, 7141596, 11724556, 22761615, -10134141}, + }, + { + FieldElement{16918416, 11729663, -18083579, 3022987, -31015732, -13339659, -28741185, -12227393, 32851222, 11717399}, + FieldElement{11166634, 7338049, -6722523, 4531520, -29468672, -7302055, 31474879, 3483633, -1193175, -4030831}, + FieldElement{-185635, 9921305, 31456609, -13536438, -12013818, 13348923, 33142652, 6546660, -19985279, -3948376}, + }, + { + FieldElement{-32460596, 11266712, -11197107, -7899103, 31703694, 3855903, -8537131, -12833048, -30772034, -15486313}, + FieldElement{-18006477, 12709068, 3991746, -6479188, -21491523, -10550425, -31135347, -16049879, 10928917, 3011958}, + FieldElement{-6957757, -15594337, 31696059, 334240, 29576716, 14796075, -30831056, -12805180, 18008031, 10258577}, + }, + { + FieldElement{-22448644, 15655569, 7018479, -4410003, -30314266, -1201591, -1853465, 1367120, 25127874, 6671743}, + FieldElement{29701166, -14373934, -10878120, 9279288, -17568, 13127210, 21382910, 11042292, 25838796, 4642684}, + FieldElement{-20430234, 14955537, -24126347, 8124619, -5369288, -5990470, 30468147, -13900640, 18423289, 4177476}, + }, + }, +} diff --git a/vendor/golang.org/x/crypto/ed25519/internal/edwards25519/edwards25519.go b/vendor/golang.org/x/crypto/ed25519/internal/edwards25519/edwards25519.go new file mode 100644 index 00000000..fd03c252 --- /dev/null +++ b/vendor/golang.org/x/crypto/ed25519/internal/edwards25519/edwards25519.go @@ -0,0 +1,1793 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package edwards25519 + +import "encoding/binary" + +// This code is a port of the public domain, “ref10” implementation of ed25519 +// from SUPERCOP. + +// FieldElement represents an element of the field GF(2^255 - 19). An element +// t, entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77 +// t[3]+2^102 t[4]+...+2^230 t[9]. Bounds on each t[i] vary depending on +// context. +type FieldElement [10]int32 + +var zero FieldElement + +func FeZero(fe *FieldElement) { + copy(fe[:], zero[:]) +} + +func FeOne(fe *FieldElement) { + FeZero(fe) + fe[0] = 1 +} + +func FeAdd(dst, a, b *FieldElement) { + dst[0] = a[0] + b[0] + dst[1] = a[1] + b[1] + dst[2] = a[2] + b[2] + dst[3] = a[3] + b[3] + dst[4] = a[4] + b[4] + dst[5] = a[5] + b[5] + dst[6] = a[6] + b[6] + dst[7] = a[7] + b[7] + dst[8] = a[8] + b[8] + dst[9] = a[9] + b[9] +} + +func FeSub(dst, a, b *FieldElement) { + dst[0] = a[0] - b[0] + dst[1] = a[1] - b[1] + dst[2] = a[2] - b[2] + dst[3] = a[3] - b[3] + dst[4] = a[4] - b[4] + dst[5] = a[5] - b[5] + dst[6] = a[6] - b[6] + dst[7] = a[7] - b[7] + dst[8] = a[8] - b[8] + dst[9] = a[9] - b[9] +} + +func FeCopy(dst, src *FieldElement) { + copy(dst[:], src[:]) +} + +// Replace (f,g) with (g,g) if b == 1; +// replace (f,g) with (f,g) if b == 0. +// +// Preconditions: b in {0,1}. +func FeCMove(f, g *FieldElement, b int32) { + b = -b + f[0] ^= b & (f[0] ^ g[0]) + f[1] ^= b & (f[1] ^ g[1]) + f[2] ^= b & (f[2] ^ g[2]) + f[3] ^= b & (f[3] ^ g[3]) + f[4] ^= b & (f[4] ^ g[4]) + f[5] ^= b & (f[5] ^ g[5]) + f[6] ^= b & (f[6] ^ g[6]) + f[7] ^= b & (f[7] ^ g[7]) + f[8] ^= b & (f[8] ^ g[8]) + f[9] ^= b & (f[9] ^ g[9]) +} + +func load3(in []byte) int64 { + var r int64 + r = int64(in[0]) + r |= int64(in[1]) << 8 + r |= int64(in[2]) << 16 + return r +} + +func load4(in []byte) int64 { + var r int64 + r = int64(in[0]) + r |= int64(in[1]) << 8 + r |= int64(in[2]) << 16 + r |= int64(in[3]) << 24 + return r +} + +func FeFromBytes(dst *FieldElement, src *[32]byte) { + h0 := load4(src[:]) + h1 := load3(src[4:]) << 6 + h2 := load3(src[7:]) << 5 + h3 := load3(src[10:]) << 3 + h4 := load3(src[13:]) << 2 + h5 := load4(src[16:]) + h6 := load3(src[20:]) << 7 + h7 := load3(src[23:]) << 5 + h8 := load3(src[26:]) << 4 + h9 := (load3(src[29:]) & 8388607) << 2 + + FeCombine(dst, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9) +} + +// FeToBytes marshals h to s. +// Preconditions: +// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. +// +// Write p=2^255-19; q=floor(h/p). +// Basic claim: q = floor(2^(-255)(h + 19 2^(-25)h9 + 2^(-1))). +// +// Proof: +// Have |h|<=p so |q|<=1 so |19^2 2^(-255) q|<1/4. +// Also have |h-2^230 h9|<2^230 so |19 2^(-255)(h-2^230 h9)|<1/4. +// +// Write y=2^(-1)-19^2 2^(-255)q-19 2^(-255)(h-2^230 h9). +// Then 0> 25 + q = (h[0] + q) >> 26 + q = (h[1] + q) >> 25 + q = (h[2] + q) >> 26 + q = (h[3] + q) >> 25 + q = (h[4] + q) >> 26 + q = (h[5] + q) >> 25 + q = (h[6] + q) >> 26 + q = (h[7] + q) >> 25 + q = (h[8] + q) >> 26 + q = (h[9] + q) >> 25 + + // Goal: Output h-(2^255-19)q, which is between 0 and 2^255-20. + h[0] += 19 * q + // Goal: Output h-2^255 q, which is between 0 and 2^255-20. + + carry[0] = h[0] >> 26 + h[1] += carry[0] + h[0] -= carry[0] << 26 + carry[1] = h[1] >> 25 + h[2] += carry[1] + h[1] -= carry[1] << 25 + carry[2] = h[2] >> 26 + h[3] += carry[2] + h[2] -= carry[2] << 26 + carry[3] = h[3] >> 25 + h[4] += carry[3] + h[3] -= carry[3] << 25 + carry[4] = h[4] >> 26 + h[5] += carry[4] + h[4] -= carry[4] << 26 + carry[5] = h[5] >> 25 + h[6] += carry[5] + h[5] -= carry[5] << 25 + carry[6] = h[6] >> 26 + h[7] += carry[6] + h[6] -= carry[6] << 26 + carry[7] = h[7] >> 25 + h[8] += carry[7] + h[7] -= carry[7] << 25 + carry[8] = h[8] >> 26 + h[9] += carry[8] + h[8] -= carry[8] << 26 + carry[9] = h[9] >> 25 + h[9] -= carry[9] << 25 + // h10 = carry9 + + // Goal: Output h[0]+...+2^255 h10-2^255 q, which is between 0 and 2^255-20. + // Have h[0]+...+2^230 h[9] between 0 and 2^255-1; + // evidently 2^255 h10-2^255 q = 0. + // Goal: Output h[0]+...+2^230 h[9]. + + s[0] = byte(h[0] >> 0) + s[1] = byte(h[0] >> 8) + s[2] = byte(h[0] >> 16) + s[3] = byte((h[0] >> 24) | (h[1] << 2)) + s[4] = byte(h[1] >> 6) + s[5] = byte(h[1] >> 14) + s[6] = byte((h[1] >> 22) | (h[2] << 3)) + s[7] = byte(h[2] >> 5) + s[8] = byte(h[2] >> 13) + s[9] = byte((h[2] >> 21) | (h[3] << 5)) + s[10] = byte(h[3] >> 3) + s[11] = byte(h[3] >> 11) + s[12] = byte((h[3] >> 19) | (h[4] << 6)) + s[13] = byte(h[4] >> 2) + s[14] = byte(h[4] >> 10) + s[15] = byte(h[4] >> 18) + s[16] = byte(h[5] >> 0) + s[17] = byte(h[5] >> 8) + s[18] = byte(h[5] >> 16) + s[19] = byte((h[5] >> 24) | (h[6] << 1)) + s[20] = byte(h[6] >> 7) + s[21] = byte(h[6] >> 15) + s[22] = byte((h[6] >> 23) | (h[7] << 3)) + s[23] = byte(h[7] >> 5) + s[24] = byte(h[7] >> 13) + s[25] = byte((h[7] >> 21) | (h[8] << 4)) + s[26] = byte(h[8] >> 4) + s[27] = byte(h[8] >> 12) + s[28] = byte((h[8] >> 20) | (h[9] << 6)) + s[29] = byte(h[9] >> 2) + s[30] = byte(h[9] >> 10) + s[31] = byte(h[9] >> 18) +} + +func FeIsNegative(f *FieldElement) byte { + var s [32]byte + FeToBytes(&s, f) + return s[0] & 1 +} + +func FeIsNonZero(f *FieldElement) int32 { + var s [32]byte + FeToBytes(&s, f) + var x uint8 + for _, b := range s { + x |= b + } + x |= x >> 4 + x |= x >> 2 + x |= x >> 1 + return int32(x & 1) +} + +// FeNeg sets h = -f +// +// Preconditions: +// |f| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. +// +// Postconditions: +// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. +func FeNeg(h, f *FieldElement) { + h[0] = -f[0] + h[1] = -f[1] + h[2] = -f[2] + h[3] = -f[3] + h[4] = -f[4] + h[5] = -f[5] + h[6] = -f[6] + h[7] = -f[7] + h[8] = -f[8] + h[9] = -f[9] +} + +func FeCombine(h *FieldElement, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9 int64) { + var c0, c1, c2, c3, c4, c5, c6, c7, c8, c9 int64 + + /* + |h0| <= (1.1*1.1*2^52*(1+19+19+19+19)+1.1*1.1*2^50*(38+38+38+38+38)) + i.e. |h0| <= 1.2*2^59; narrower ranges for h2, h4, h6, h8 + |h1| <= (1.1*1.1*2^51*(1+1+19+19+19+19+19+19+19+19)) + i.e. |h1| <= 1.5*2^58; narrower ranges for h3, h5, h7, h9 + */ + + c0 = (h0 + (1 << 25)) >> 26 + h1 += c0 + h0 -= c0 << 26 + c4 = (h4 + (1 << 25)) >> 26 + h5 += c4 + h4 -= c4 << 26 + /* |h0| <= 2^25 */ + /* |h4| <= 2^25 */ + /* |h1| <= 1.51*2^58 */ + /* |h5| <= 1.51*2^58 */ + + c1 = (h1 + (1 << 24)) >> 25 + h2 += c1 + h1 -= c1 << 25 + c5 = (h5 + (1 << 24)) >> 25 + h6 += c5 + h5 -= c5 << 25 + /* |h1| <= 2^24; from now on fits into int32 */ + /* |h5| <= 2^24; from now on fits into int32 */ + /* |h2| <= 1.21*2^59 */ + /* |h6| <= 1.21*2^59 */ + + c2 = (h2 + (1 << 25)) >> 26 + h3 += c2 + h2 -= c2 << 26 + c6 = (h6 + (1 << 25)) >> 26 + h7 += c6 + h6 -= c6 << 26 + /* |h2| <= 2^25; from now on fits into int32 unchanged */ + /* |h6| <= 2^25; from now on fits into int32 unchanged */ + /* |h3| <= 1.51*2^58 */ + /* |h7| <= 1.51*2^58 */ + + c3 = (h3 + (1 << 24)) >> 25 + h4 += c3 + h3 -= c3 << 25 + c7 = (h7 + (1 << 24)) >> 25 + h8 += c7 + h7 -= c7 << 25 + /* |h3| <= 2^24; from now on fits into int32 unchanged */ + /* |h7| <= 2^24; from now on fits into int32 unchanged */ + /* |h4| <= 1.52*2^33 */ + /* |h8| <= 1.52*2^33 */ + + c4 = (h4 + (1 << 25)) >> 26 + h5 += c4 + h4 -= c4 << 26 + c8 = (h8 + (1 << 25)) >> 26 + h9 += c8 + h8 -= c8 << 26 + /* |h4| <= 2^25; from now on fits into int32 unchanged */ + /* |h8| <= 2^25; from now on fits into int32 unchanged */ + /* |h5| <= 1.01*2^24 */ + /* |h9| <= 1.51*2^58 */ + + c9 = (h9 + (1 << 24)) >> 25 + h0 += c9 * 19 + h9 -= c9 << 25 + /* |h9| <= 2^24; from now on fits into int32 unchanged */ + /* |h0| <= 1.8*2^37 */ + + c0 = (h0 + (1 << 25)) >> 26 + h1 += c0 + h0 -= c0 << 26 + /* |h0| <= 2^25; from now on fits into int32 unchanged */ + /* |h1| <= 1.01*2^24 */ + + h[0] = int32(h0) + h[1] = int32(h1) + h[2] = int32(h2) + h[3] = int32(h3) + h[4] = int32(h4) + h[5] = int32(h5) + h[6] = int32(h6) + h[7] = int32(h7) + h[8] = int32(h8) + h[9] = int32(h9) +} + +// FeMul calculates h = f * g +// Can overlap h with f or g. +// +// Preconditions: +// |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. +// |g| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. +// +// Postconditions: +// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. +// +// Notes on implementation strategy: +// +// Using schoolbook multiplication. +// Karatsuba would save a little in some cost models. +// +// Most multiplications by 2 and 19 are 32-bit precomputations; +// cheaper than 64-bit postcomputations. +// +// There is one remaining multiplication by 19 in the carry chain; +// one *19 precomputation can be merged into this, +// but the resulting data flow is considerably less clean. +// +// There are 12 carries below. +// 10 of them are 2-way parallelizable and vectorizable. +// Can get away with 11 carries, but then data flow is much deeper. +// +// With tighter constraints on inputs, can squeeze carries into int32. +func FeMul(h, f, g *FieldElement) { + f0 := int64(f[0]) + f1 := int64(f[1]) + f2 := int64(f[2]) + f3 := int64(f[3]) + f4 := int64(f[4]) + f5 := int64(f[5]) + f6 := int64(f[6]) + f7 := int64(f[7]) + f8 := int64(f[8]) + f9 := int64(f[9]) + + f1_2 := int64(2 * f[1]) + f3_2 := int64(2 * f[3]) + f5_2 := int64(2 * f[5]) + f7_2 := int64(2 * f[7]) + f9_2 := int64(2 * f[9]) + + g0 := int64(g[0]) + g1 := int64(g[1]) + g2 := int64(g[2]) + g3 := int64(g[3]) + g4 := int64(g[4]) + g5 := int64(g[5]) + g6 := int64(g[6]) + g7 := int64(g[7]) + g8 := int64(g[8]) + g9 := int64(g[9]) + + g1_19 := int64(19 * g[1]) /* 1.4*2^29 */ + g2_19 := int64(19 * g[2]) /* 1.4*2^30; still ok */ + g3_19 := int64(19 * g[3]) + g4_19 := int64(19 * g[4]) + g5_19 := int64(19 * g[5]) + g6_19 := int64(19 * g[6]) + g7_19 := int64(19 * g[7]) + g8_19 := int64(19 * g[8]) + g9_19 := int64(19 * g[9]) + + h0 := f0*g0 + f1_2*g9_19 + f2*g8_19 + f3_2*g7_19 + f4*g6_19 + f5_2*g5_19 + f6*g4_19 + f7_2*g3_19 + f8*g2_19 + f9_2*g1_19 + h1 := f0*g1 + f1*g0 + f2*g9_19 + f3*g8_19 + f4*g7_19 + f5*g6_19 + f6*g5_19 + f7*g4_19 + f8*g3_19 + f9*g2_19 + h2 := f0*g2 + f1_2*g1 + f2*g0 + f3_2*g9_19 + f4*g8_19 + f5_2*g7_19 + f6*g6_19 + f7_2*g5_19 + f8*g4_19 + f9_2*g3_19 + h3 := f0*g3 + f1*g2 + f2*g1 + f3*g0 + f4*g9_19 + f5*g8_19 + f6*g7_19 + f7*g6_19 + f8*g5_19 + f9*g4_19 + h4 := f0*g4 + f1_2*g3 + f2*g2 + f3_2*g1 + f4*g0 + f5_2*g9_19 + f6*g8_19 + f7_2*g7_19 + f8*g6_19 + f9_2*g5_19 + h5 := f0*g5 + f1*g4 + f2*g3 + f3*g2 + f4*g1 + f5*g0 + f6*g9_19 + f7*g8_19 + f8*g7_19 + f9*g6_19 + h6 := f0*g6 + f1_2*g5 + f2*g4 + f3_2*g3 + f4*g2 + f5_2*g1 + f6*g0 + f7_2*g9_19 + f8*g8_19 + f9_2*g7_19 + h7 := f0*g7 + f1*g6 + f2*g5 + f3*g4 + f4*g3 + f5*g2 + f6*g1 + f7*g0 + f8*g9_19 + f9*g8_19 + h8 := f0*g8 + f1_2*g7 + f2*g6 + f3_2*g5 + f4*g4 + f5_2*g3 + f6*g2 + f7_2*g1 + f8*g0 + f9_2*g9_19 + h9 := f0*g9 + f1*g8 + f2*g7 + f3*g6 + f4*g5 + f5*g4 + f6*g3 + f7*g2 + f8*g1 + f9*g0 + + FeCombine(h, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9) +} + +func feSquare(f *FieldElement) (h0, h1, h2, h3, h4, h5, h6, h7, h8, h9 int64) { + f0 := int64(f[0]) + f1 := int64(f[1]) + f2 := int64(f[2]) + f3 := int64(f[3]) + f4 := int64(f[4]) + f5 := int64(f[5]) + f6 := int64(f[6]) + f7 := int64(f[7]) + f8 := int64(f[8]) + f9 := int64(f[9]) + f0_2 := int64(2 * f[0]) + f1_2 := int64(2 * f[1]) + f2_2 := int64(2 * f[2]) + f3_2 := int64(2 * f[3]) + f4_2 := int64(2 * f[4]) + f5_2 := int64(2 * f[5]) + f6_2 := int64(2 * f[6]) + f7_2 := int64(2 * f[7]) + f5_38 := 38 * f5 // 1.31*2^30 + f6_19 := 19 * f6 // 1.31*2^30 + f7_38 := 38 * f7 // 1.31*2^30 + f8_19 := 19 * f8 // 1.31*2^30 + f9_38 := 38 * f9 // 1.31*2^30 + + h0 = f0*f0 + f1_2*f9_38 + f2_2*f8_19 + f3_2*f7_38 + f4_2*f6_19 + f5*f5_38 + h1 = f0_2*f1 + f2*f9_38 + f3_2*f8_19 + f4*f7_38 + f5_2*f6_19 + h2 = f0_2*f2 + f1_2*f1 + f3_2*f9_38 + f4_2*f8_19 + f5_2*f7_38 + f6*f6_19 + h3 = f0_2*f3 + f1_2*f2 + f4*f9_38 + f5_2*f8_19 + f6*f7_38 + h4 = f0_2*f4 + f1_2*f3_2 + f2*f2 + f5_2*f9_38 + f6_2*f8_19 + f7*f7_38 + h5 = f0_2*f5 + f1_2*f4 + f2_2*f3 + f6*f9_38 + f7_2*f8_19 + h6 = f0_2*f6 + f1_2*f5_2 + f2_2*f4 + f3_2*f3 + f7_2*f9_38 + f8*f8_19 + h7 = f0_2*f7 + f1_2*f6 + f2_2*f5 + f3_2*f4 + f8*f9_38 + h8 = f0_2*f8 + f1_2*f7_2 + f2_2*f6 + f3_2*f5_2 + f4*f4 + f9*f9_38 + h9 = f0_2*f9 + f1_2*f8 + f2_2*f7 + f3_2*f6 + f4_2*f5 + + return +} + +// FeSquare calculates h = f*f. Can overlap h with f. +// +// Preconditions: +// |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. +// +// Postconditions: +// |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. +func FeSquare(h, f *FieldElement) { + h0, h1, h2, h3, h4, h5, h6, h7, h8, h9 := feSquare(f) + FeCombine(h, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9) +} + +// FeSquare2 sets h = 2 * f * f +// +// Can overlap h with f. +// +// Preconditions: +// |f| bounded by 1.65*2^26,1.65*2^25,1.65*2^26,1.65*2^25,etc. +// +// Postconditions: +// |h| bounded by 1.01*2^25,1.01*2^24,1.01*2^25,1.01*2^24,etc. +// See fe_mul.c for discussion of implementation strategy. +func FeSquare2(h, f *FieldElement) { + h0, h1, h2, h3, h4, h5, h6, h7, h8, h9 := feSquare(f) + + h0 += h0 + h1 += h1 + h2 += h2 + h3 += h3 + h4 += h4 + h5 += h5 + h6 += h6 + h7 += h7 + h8 += h8 + h9 += h9 + + FeCombine(h, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9) +} + +func FeInvert(out, z *FieldElement) { + var t0, t1, t2, t3 FieldElement + var i int + + FeSquare(&t0, z) // 2^1 + FeSquare(&t1, &t0) // 2^2 + for i = 1; i < 2; i++ { // 2^3 + FeSquare(&t1, &t1) + } + FeMul(&t1, z, &t1) // 2^3 + 2^0 + FeMul(&t0, &t0, &t1) // 2^3 + 2^1 + 2^0 + FeSquare(&t2, &t0) // 2^4 + 2^2 + 2^1 + FeMul(&t1, &t1, &t2) // 2^4 + 2^3 + 2^2 + 2^1 + 2^0 + FeSquare(&t2, &t1) // 5,4,3,2,1 + for i = 1; i < 5; i++ { // 9,8,7,6,5 + FeSquare(&t2, &t2) + } + FeMul(&t1, &t2, &t1) // 9,8,7,6,5,4,3,2,1,0 + FeSquare(&t2, &t1) // 10..1 + for i = 1; i < 10; i++ { // 19..10 + FeSquare(&t2, &t2) + } + FeMul(&t2, &t2, &t1) // 19..0 + FeSquare(&t3, &t2) // 20..1 + for i = 1; i < 20; i++ { // 39..20 + FeSquare(&t3, &t3) + } + FeMul(&t2, &t3, &t2) // 39..0 + FeSquare(&t2, &t2) // 40..1 + for i = 1; i < 10; i++ { // 49..10 + FeSquare(&t2, &t2) + } + FeMul(&t1, &t2, &t1) // 49..0 + FeSquare(&t2, &t1) // 50..1 + for i = 1; i < 50; i++ { // 99..50 + FeSquare(&t2, &t2) + } + FeMul(&t2, &t2, &t1) // 99..0 + FeSquare(&t3, &t2) // 100..1 + for i = 1; i < 100; i++ { // 199..100 + FeSquare(&t3, &t3) + } + FeMul(&t2, &t3, &t2) // 199..0 + FeSquare(&t2, &t2) // 200..1 + for i = 1; i < 50; i++ { // 249..50 + FeSquare(&t2, &t2) + } + FeMul(&t1, &t2, &t1) // 249..0 + FeSquare(&t1, &t1) // 250..1 + for i = 1; i < 5; i++ { // 254..5 + FeSquare(&t1, &t1) + } + FeMul(out, &t1, &t0) // 254..5,3,1,0 +} + +func fePow22523(out, z *FieldElement) { + var t0, t1, t2 FieldElement + var i int + + FeSquare(&t0, z) + for i = 1; i < 1; i++ { + FeSquare(&t0, &t0) + } + FeSquare(&t1, &t0) + for i = 1; i < 2; i++ { + FeSquare(&t1, &t1) + } + FeMul(&t1, z, &t1) + FeMul(&t0, &t0, &t1) + FeSquare(&t0, &t0) + for i = 1; i < 1; i++ { + FeSquare(&t0, &t0) + } + FeMul(&t0, &t1, &t0) + FeSquare(&t1, &t0) + for i = 1; i < 5; i++ { + FeSquare(&t1, &t1) + } + FeMul(&t0, &t1, &t0) + FeSquare(&t1, &t0) + for i = 1; i < 10; i++ { + FeSquare(&t1, &t1) + } + FeMul(&t1, &t1, &t0) + FeSquare(&t2, &t1) + for i = 1; i < 20; i++ { + FeSquare(&t2, &t2) + } + FeMul(&t1, &t2, &t1) + FeSquare(&t1, &t1) + for i = 1; i < 10; i++ { + FeSquare(&t1, &t1) + } + FeMul(&t0, &t1, &t0) + FeSquare(&t1, &t0) + for i = 1; i < 50; i++ { + FeSquare(&t1, &t1) + } + FeMul(&t1, &t1, &t0) + FeSquare(&t2, &t1) + for i = 1; i < 100; i++ { + FeSquare(&t2, &t2) + } + FeMul(&t1, &t2, &t1) + FeSquare(&t1, &t1) + for i = 1; i < 50; i++ { + FeSquare(&t1, &t1) + } + FeMul(&t0, &t1, &t0) + FeSquare(&t0, &t0) + for i = 1; i < 2; i++ { + FeSquare(&t0, &t0) + } + FeMul(out, &t0, z) +} + +// Group elements are members of the elliptic curve -x^2 + y^2 = 1 + d * x^2 * +// y^2 where d = -121665/121666. +// +// Several representations are used: +// ProjectiveGroupElement: (X:Y:Z) satisfying x=X/Z, y=Y/Z +// ExtendedGroupElement: (X:Y:Z:T) satisfying x=X/Z, y=Y/Z, XY=ZT +// CompletedGroupElement: ((X:Z),(Y:T)) satisfying x=X/Z, y=Y/T +// PreComputedGroupElement: (y+x,y-x,2dxy) + +type ProjectiveGroupElement struct { + X, Y, Z FieldElement +} + +type ExtendedGroupElement struct { + X, Y, Z, T FieldElement +} + +type CompletedGroupElement struct { + X, Y, Z, T FieldElement +} + +type PreComputedGroupElement struct { + yPlusX, yMinusX, xy2d FieldElement +} + +type CachedGroupElement struct { + yPlusX, yMinusX, Z, T2d FieldElement +} + +func (p *ProjectiveGroupElement) Zero() { + FeZero(&p.X) + FeOne(&p.Y) + FeOne(&p.Z) +} + +func (p *ProjectiveGroupElement) Double(r *CompletedGroupElement) { + var t0 FieldElement + + FeSquare(&r.X, &p.X) + FeSquare(&r.Z, &p.Y) + FeSquare2(&r.T, &p.Z) + FeAdd(&r.Y, &p.X, &p.Y) + FeSquare(&t0, &r.Y) + FeAdd(&r.Y, &r.Z, &r.X) + FeSub(&r.Z, &r.Z, &r.X) + FeSub(&r.X, &t0, &r.Y) + FeSub(&r.T, &r.T, &r.Z) +} + +func (p *ProjectiveGroupElement) ToBytes(s *[32]byte) { + var recip, x, y FieldElement + + FeInvert(&recip, &p.Z) + FeMul(&x, &p.X, &recip) + FeMul(&y, &p.Y, &recip) + FeToBytes(s, &y) + s[31] ^= FeIsNegative(&x) << 7 +} + +func (p *ExtendedGroupElement) Zero() { + FeZero(&p.X) + FeOne(&p.Y) + FeOne(&p.Z) + FeZero(&p.T) +} + +func (p *ExtendedGroupElement) Double(r *CompletedGroupElement) { + var q ProjectiveGroupElement + p.ToProjective(&q) + q.Double(r) +} + +func (p *ExtendedGroupElement) ToCached(r *CachedGroupElement) { + FeAdd(&r.yPlusX, &p.Y, &p.X) + FeSub(&r.yMinusX, &p.Y, &p.X) + FeCopy(&r.Z, &p.Z) + FeMul(&r.T2d, &p.T, &d2) +} + +func (p *ExtendedGroupElement) ToProjective(r *ProjectiveGroupElement) { + FeCopy(&r.X, &p.X) + FeCopy(&r.Y, &p.Y) + FeCopy(&r.Z, &p.Z) +} + +func (p *ExtendedGroupElement) ToBytes(s *[32]byte) { + var recip, x, y FieldElement + + FeInvert(&recip, &p.Z) + FeMul(&x, &p.X, &recip) + FeMul(&y, &p.Y, &recip) + FeToBytes(s, &y) + s[31] ^= FeIsNegative(&x) << 7 +} + +func (p *ExtendedGroupElement) FromBytes(s *[32]byte) bool { + var u, v, v3, vxx, check FieldElement + + FeFromBytes(&p.Y, s) + FeOne(&p.Z) + FeSquare(&u, &p.Y) + FeMul(&v, &u, &d) + FeSub(&u, &u, &p.Z) // y = y^2-1 + FeAdd(&v, &v, &p.Z) // v = dy^2+1 + + FeSquare(&v3, &v) + FeMul(&v3, &v3, &v) // v3 = v^3 + FeSquare(&p.X, &v3) + FeMul(&p.X, &p.X, &v) + FeMul(&p.X, &p.X, &u) // x = uv^7 + + fePow22523(&p.X, &p.X) // x = (uv^7)^((q-5)/8) + FeMul(&p.X, &p.X, &v3) + FeMul(&p.X, &p.X, &u) // x = uv^3(uv^7)^((q-5)/8) + + var tmpX, tmp2 [32]byte + + FeSquare(&vxx, &p.X) + FeMul(&vxx, &vxx, &v) + FeSub(&check, &vxx, &u) // vx^2-u + if FeIsNonZero(&check) == 1 { + FeAdd(&check, &vxx, &u) // vx^2+u + if FeIsNonZero(&check) == 1 { + return false + } + FeMul(&p.X, &p.X, &SqrtM1) + + FeToBytes(&tmpX, &p.X) + for i, v := range tmpX { + tmp2[31-i] = v + } + } + + if FeIsNegative(&p.X) != (s[31] >> 7) { + FeNeg(&p.X, &p.X) + } + + FeMul(&p.T, &p.X, &p.Y) + return true +} + +func (p *CompletedGroupElement) ToProjective(r *ProjectiveGroupElement) { + FeMul(&r.X, &p.X, &p.T) + FeMul(&r.Y, &p.Y, &p.Z) + FeMul(&r.Z, &p.Z, &p.T) +} + +func (p *CompletedGroupElement) ToExtended(r *ExtendedGroupElement) { + FeMul(&r.X, &p.X, &p.T) + FeMul(&r.Y, &p.Y, &p.Z) + FeMul(&r.Z, &p.Z, &p.T) + FeMul(&r.T, &p.X, &p.Y) +} + +func (p *PreComputedGroupElement) Zero() { + FeOne(&p.yPlusX) + FeOne(&p.yMinusX) + FeZero(&p.xy2d) +} + +func geAdd(r *CompletedGroupElement, p *ExtendedGroupElement, q *CachedGroupElement) { + var t0 FieldElement + + FeAdd(&r.X, &p.Y, &p.X) + FeSub(&r.Y, &p.Y, &p.X) + FeMul(&r.Z, &r.X, &q.yPlusX) + FeMul(&r.Y, &r.Y, &q.yMinusX) + FeMul(&r.T, &q.T2d, &p.T) + FeMul(&r.X, &p.Z, &q.Z) + FeAdd(&t0, &r.X, &r.X) + FeSub(&r.X, &r.Z, &r.Y) + FeAdd(&r.Y, &r.Z, &r.Y) + FeAdd(&r.Z, &t0, &r.T) + FeSub(&r.T, &t0, &r.T) +} + +func geSub(r *CompletedGroupElement, p *ExtendedGroupElement, q *CachedGroupElement) { + var t0 FieldElement + + FeAdd(&r.X, &p.Y, &p.X) + FeSub(&r.Y, &p.Y, &p.X) + FeMul(&r.Z, &r.X, &q.yMinusX) + FeMul(&r.Y, &r.Y, &q.yPlusX) + FeMul(&r.T, &q.T2d, &p.T) + FeMul(&r.X, &p.Z, &q.Z) + FeAdd(&t0, &r.X, &r.X) + FeSub(&r.X, &r.Z, &r.Y) + FeAdd(&r.Y, &r.Z, &r.Y) + FeSub(&r.Z, &t0, &r.T) + FeAdd(&r.T, &t0, &r.T) +} + +func geMixedAdd(r *CompletedGroupElement, p *ExtendedGroupElement, q *PreComputedGroupElement) { + var t0 FieldElement + + FeAdd(&r.X, &p.Y, &p.X) + FeSub(&r.Y, &p.Y, &p.X) + FeMul(&r.Z, &r.X, &q.yPlusX) + FeMul(&r.Y, &r.Y, &q.yMinusX) + FeMul(&r.T, &q.xy2d, &p.T) + FeAdd(&t0, &p.Z, &p.Z) + FeSub(&r.X, &r.Z, &r.Y) + FeAdd(&r.Y, &r.Z, &r.Y) + FeAdd(&r.Z, &t0, &r.T) + FeSub(&r.T, &t0, &r.T) +} + +func geMixedSub(r *CompletedGroupElement, p *ExtendedGroupElement, q *PreComputedGroupElement) { + var t0 FieldElement + + FeAdd(&r.X, &p.Y, &p.X) + FeSub(&r.Y, &p.Y, &p.X) + FeMul(&r.Z, &r.X, &q.yMinusX) + FeMul(&r.Y, &r.Y, &q.yPlusX) + FeMul(&r.T, &q.xy2d, &p.T) + FeAdd(&t0, &p.Z, &p.Z) + FeSub(&r.X, &r.Z, &r.Y) + FeAdd(&r.Y, &r.Z, &r.Y) + FeSub(&r.Z, &t0, &r.T) + FeAdd(&r.T, &t0, &r.T) +} + +func slide(r *[256]int8, a *[32]byte) { + for i := range r { + r[i] = int8(1 & (a[i>>3] >> uint(i&7))) + } + + for i := range r { + if r[i] != 0 { + for b := 1; b <= 6 && i+b < 256; b++ { + if r[i+b] != 0 { + if r[i]+(r[i+b]<= -15 { + r[i] -= r[i+b] << uint(b) + for k := i + b; k < 256; k++ { + if r[k] == 0 { + r[k] = 1 + break + } + r[k] = 0 + } + } else { + break + } + } + } + } + } +} + +// GeDoubleScalarMultVartime sets r = a*A + b*B +// where a = a[0]+256*a[1]+...+256^31 a[31]. +// and b = b[0]+256*b[1]+...+256^31 b[31]. +// B is the Ed25519 base point (x,4/5) with x positive. +func GeDoubleScalarMultVartime(r *ProjectiveGroupElement, a *[32]byte, A *ExtendedGroupElement, b *[32]byte) { + var aSlide, bSlide [256]int8 + var Ai [8]CachedGroupElement // A,3A,5A,7A,9A,11A,13A,15A + var t CompletedGroupElement + var u, A2 ExtendedGroupElement + var i int + + slide(&aSlide, a) + slide(&bSlide, b) + + A.ToCached(&Ai[0]) + A.Double(&t) + t.ToExtended(&A2) + + for i := 0; i < 7; i++ { + geAdd(&t, &A2, &Ai[i]) + t.ToExtended(&u) + u.ToCached(&Ai[i+1]) + } + + r.Zero() + + for i = 255; i >= 0; i-- { + if aSlide[i] != 0 || bSlide[i] != 0 { + break + } + } + + for ; i >= 0; i-- { + r.Double(&t) + + if aSlide[i] > 0 { + t.ToExtended(&u) + geAdd(&t, &u, &Ai[aSlide[i]/2]) + } else if aSlide[i] < 0 { + t.ToExtended(&u) + geSub(&t, &u, &Ai[(-aSlide[i])/2]) + } + + if bSlide[i] > 0 { + t.ToExtended(&u) + geMixedAdd(&t, &u, &bi[bSlide[i]/2]) + } else if bSlide[i] < 0 { + t.ToExtended(&u) + geMixedSub(&t, &u, &bi[(-bSlide[i])/2]) + } + + t.ToProjective(r) + } +} + +// equal returns 1 if b == c and 0 otherwise, assuming that b and c are +// non-negative. +func equal(b, c int32) int32 { + x := uint32(b ^ c) + x-- + return int32(x >> 31) +} + +// negative returns 1 if b < 0 and 0 otherwise. +func negative(b int32) int32 { + return (b >> 31) & 1 +} + +func PreComputedGroupElementCMove(t, u *PreComputedGroupElement, b int32) { + FeCMove(&t.yPlusX, &u.yPlusX, b) + FeCMove(&t.yMinusX, &u.yMinusX, b) + FeCMove(&t.xy2d, &u.xy2d, b) +} + +func selectPoint(t *PreComputedGroupElement, pos int32, b int32) { + var minusT PreComputedGroupElement + bNegative := negative(b) + bAbs := b - (((-bNegative) & b) << 1) + + t.Zero() + for i := int32(0); i < 8; i++ { + PreComputedGroupElementCMove(t, &base[pos][i], equal(bAbs, i+1)) + } + FeCopy(&minusT.yPlusX, &t.yMinusX) + FeCopy(&minusT.yMinusX, &t.yPlusX) + FeNeg(&minusT.xy2d, &t.xy2d) + PreComputedGroupElementCMove(t, &minusT, bNegative) +} + +// GeScalarMultBase computes h = a*B, where +// a = a[0]+256*a[1]+...+256^31 a[31] +// B is the Ed25519 base point (x,4/5) with x positive. +// +// Preconditions: +// a[31] <= 127 +func GeScalarMultBase(h *ExtendedGroupElement, a *[32]byte) { + var e [64]int8 + + for i, v := range a { + e[2*i] = int8(v & 15) + e[2*i+1] = int8((v >> 4) & 15) + } + + // each e[i] is between 0 and 15 and e[63] is between 0 and 7. + + carry := int8(0) + for i := 0; i < 63; i++ { + e[i] += carry + carry = (e[i] + 8) >> 4 + e[i] -= carry << 4 + } + e[63] += carry + // each e[i] is between -8 and 8. + + h.Zero() + var t PreComputedGroupElement + var r CompletedGroupElement + for i := int32(1); i < 64; i += 2 { + selectPoint(&t, i/2, int32(e[i])) + geMixedAdd(&r, h, &t) + r.ToExtended(h) + } + + var s ProjectiveGroupElement + + h.Double(&r) + r.ToProjective(&s) + s.Double(&r) + r.ToProjective(&s) + s.Double(&r) + r.ToProjective(&s) + s.Double(&r) + r.ToExtended(h) + + for i := int32(0); i < 64; i += 2 { + selectPoint(&t, i/2, int32(e[i])) + geMixedAdd(&r, h, &t) + r.ToExtended(h) + } +} + +// The scalars are GF(2^252 + 27742317777372353535851937790883648493). + +// Input: +// a[0]+256*a[1]+...+256^31*a[31] = a +// b[0]+256*b[1]+...+256^31*b[31] = b +// c[0]+256*c[1]+...+256^31*c[31] = c +// +// Output: +// s[0]+256*s[1]+...+256^31*s[31] = (ab+c) mod l +// where l = 2^252 + 27742317777372353535851937790883648493. +func ScMulAdd(s, a, b, c *[32]byte) { + a0 := 2097151 & load3(a[:]) + a1 := 2097151 & (load4(a[2:]) >> 5) + a2 := 2097151 & (load3(a[5:]) >> 2) + a3 := 2097151 & (load4(a[7:]) >> 7) + a4 := 2097151 & (load4(a[10:]) >> 4) + a5 := 2097151 & (load3(a[13:]) >> 1) + a6 := 2097151 & (load4(a[15:]) >> 6) + a7 := 2097151 & (load3(a[18:]) >> 3) + a8 := 2097151 & load3(a[21:]) + a9 := 2097151 & (load4(a[23:]) >> 5) + a10 := 2097151 & (load3(a[26:]) >> 2) + a11 := (load4(a[28:]) >> 7) + b0 := 2097151 & load3(b[:]) + b1 := 2097151 & (load4(b[2:]) >> 5) + b2 := 2097151 & (load3(b[5:]) >> 2) + b3 := 2097151 & (load4(b[7:]) >> 7) + b4 := 2097151 & (load4(b[10:]) >> 4) + b5 := 2097151 & (load3(b[13:]) >> 1) + b6 := 2097151 & (load4(b[15:]) >> 6) + b7 := 2097151 & (load3(b[18:]) >> 3) + b8 := 2097151 & load3(b[21:]) + b9 := 2097151 & (load4(b[23:]) >> 5) + b10 := 2097151 & (load3(b[26:]) >> 2) + b11 := (load4(b[28:]) >> 7) + c0 := 2097151 & load3(c[:]) + c1 := 2097151 & (load4(c[2:]) >> 5) + c2 := 2097151 & (load3(c[5:]) >> 2) + c3 := 2097151 & (load4(c[7:]) >> 7) + c4 := 2097151 & (load4(c[10:]) >> 4) + c5 := 2097151 & (load3(c[13:]) >> 1) + c6 := 2097151 & (load4(c[15:]) >> 6) + c7 := 2097151 & (load3(c[18:]) >> 3) + c8 := 2097151 & load3(c[21:]) + c9 := 2097151 & (load4(c[23:]) >> 5) + c10 := 2097151 & (load3(c[26:]) >> 2) + c11 := (load4(c[28:]) >> 7) + var carry [23]int64 + + s0 := c0 + a0*b0 + s1 := c1 + a0*b1 + a1*b0 + s2 := c2 + a0*b2 + a1*b1 + a2*b0 + s3 := c3 + a0*b3 + a1*b2 + a2*b1 + a3*b0 + s4 := c4 + a0*b4 + a1*b3 + a2*b2 + a3*b1 + a4*b0 + s5 := c5 + a0*b5 + a1*b4 + a2*b3 + a3*b2 + a4*b1 + a5*b0 + s6 := c6 + a0*b6 + a1*b5 + a2*b4 + a3*b3 + a4*b2 + a5*b1 + a6*b0 + s7 := c7 + a0*b7 + a1*b6 + a2*b5 + a3*b4 + a4*b3 + a5*b2 + a6*b1 + a7*b0 + s8 := c8 + a0*b8 + a1*b7 + a2*b6 + a3*b5 + a4*b4 + a5*b3 + a6*b2 + a7*b1 + a8*b0 + s9 := c9 + a0*b9 + a1*b8 + a2*b7 + a3*b6 + a4*b5 + a5*b4 + a6*b3 + a7*b2 + a8*b1 + a9*b0 + s10 := c10 + a0*b10 + a1*b9 + a2*b8 + a3*b7 + a4*b6 + a5*b5 + a6*b4 + a7*b3 + a8*b2 + a9*b1 + a10*b0 + s11 := c11 + a0*b11 + a1*b10 + a2*b9 + a3*b8 + a4*b7 + a5*b6 + a6*b5 + a7*b4 + a8*b3 + a9*b2 + a10*b1 + a11*b0 + s12 := a1*b11 + a2*b10 + a3*b9 + a4*b8 + a5*b7 + a6*b6 + a7*b5 + a8*b4 + a9*b3 + a10*b2 + a11*b1 + s13 := a2*b11 + a3*b10 + a4*b9 + a5*b8 + a6*b7 + a7*b6 + a8*b5 + a9*b4 + a10*b3 + a11*b2 + s14 := a3*b11 + a4*b10 + a5*b9 + a6*b8 + a7*b7 + a8*b6 + a9*b5 + a10*b4 + a11*b3 + s15 := a4*b11 + a5*b10 + a6*b9 + a7*b8 + a8*b7 + a9*b6 + a10*b5 + a11*b4 + s16 := a5*b11 + a6*b10 + a7*b9 + a8*b8 + a9*b7 + a10*b6 + a11*b5 + s17 := a6*b11 + a7*b10 + a8*b9 + a9*b8 + a10*b7 + a11*b6 + s18 := a7*b11 + a8*b10 + a9*b9 + a10*b8 + a11*b7 + s19 := a8*b11 + a9*b10 + a10*b9 + a11*b8 + s20 := a9*b11 + a10*b10 + a11*b9 + s21 := a10*b11 + a11*b10 + s22 := a11 * b11 + s23 := int64(0) + + carry[0] = (s0 + (1 << 20)) >> 21 + s1 += carry[0] + s0 -= carry[0] << 21 + carry[2] = (s2 + (1 << 20)) >> 21 + s3 += carry[2] + s2 -= carry[2] << 21 + carry[4] = (s4 + (1 << 20)) >> 21 + s5 += carry[4] + s4 -= carry[4] << 21 + carry[6] = (s6 + (1 << 20)) >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[8] = (s8 + (1 << 20)) >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[10] = (s10 + (1 << 20)) >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + carry[12] = (s12 + (1 << 20)) >> 21 + s13 += carry[12] + s12 -= carry[12] << 21 + carry[14] = (s14 + (1 << 20)) >> 21 + s15 += carry[14] + s14 -= carry[14] << 21 + carry[16] = (s16 + (1 << 20)) >> 21 + s17 += carry[16] + s16 -= carry[16] << 21 + carry[18] = (s18 + (1 << 20)) >> 21 + s19 += carry[18] + s18 -= carry[18] << 21 + carry[20] = (s20 + (1 << 20)) >> 21 + s21 += carry[20] + s20 -= carry[20] << 21 + carry[22] = (s22 + (1 << 20)) >> 21 + s23 += carry[22] + s22 -= carry[22] << 21 + + carry[1] = (s1 + (1 << 20)) >> 21 + s2 += carry[1] + s1 -= carry[1] << 21 + carry[3] = (s3 + (1 << 20)) >> 21 + s4 += carry[3] + s3 -= carry[3] << 21 + carry[5] = (s5 + (1 << 20)) >> 21 + s6 += carry[5] + s5 -= carry[5] << 21 + carry[7] = (s7 + (1 << 20)) >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[9] = (s9 + (1 << 20)) >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[11] = (s11 + (1 << 20)) >> 21 + s12 += carry[11] + s11 -= carry[11] << 21 + carry[13] = (s13 + (1 << 20)) >> 21 + s14 += carry[13] + s13 -= carry[13] << 21 + carry[15] = (s15 + (1 << 20)) >> 21 + s16 += carry[15] + s15 -= carry[15] << 21 + carry[17] = (s17 + (1 << 20)) >> 21 + s18 += carry[17] + s17 -= carry[17] << 21 + carry[19] = (s19 + (1 << 20)) >> 21 + s20 += carry[19] + s19 -= carry[19] << 21 + carry[21] = (s21 + (1 << 20)) >> 21 + s22 += carry[21] + s21 -= carry[21] << 21 + + s11 += s23 * 666643 + s12 += s23 * 470296 + s13 += s23 * 654183 + s14 -= s23 * 997805 + s15 += s23 * 136657 + s16 -= s23 * 683901 + s23 = 0 + + s10 += s22 * 666643 + s11 += s22 * 470296 + s12 += s22 * 654183 + s13 -= s22 * 997805 + s14 += s22 * 136657 + s15 -= s22 * 683901 + s22 = 0 + + s9 += s21 * 666643 + s10 += s21 * 470296 + s11 += s21 * 654183 + s12 -= s21 * 997805 + s13 += s21 * 136657 + s14 -= s21 * 683901 + s21 = 0 + + s8 += s20 * 666643 + s9 += s20 * 470296 + s10 += s20 * 654183 + s11 -= s20 * 997805 + s12 += s20 * 136657 + s13 -= s20 * 683901 + s20 = 0 + + s7 += s19 * 666643 + s8 += s19 * 470296 + s9 += s19 * 654183 + s10 -= s19 * 997805 + s11 += s19 * 136657 + s12 -= s19 * 683901 + s19 = 0 + + s6 += s18 * 666643 + s7 += s18 * 470296 + s8 += s18 * 654183 + s9 -= s18 * 997805 + s10 += s18 * 136657 + s11 -= s18 * 683901 + s18 = 0 + + carry[6] = (s6 + (1 << 20)) >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[8] = (s8 + (1 << 20)) >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[10] = (s10 + (1 << 20)) >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + carry[12] = (s12 + (1 << 20)) >> 21 + s13 += carry[12] + s12 -= carry[12] << 21 + carry[14] = (s14 + (1 << 20)) >> 21 + s15 += carry[14] + s14 -= carry[14] << 21 + carry[16] = (s16 + (1 << 20)) >> 21 + s17 += carry[16] + s16 -= carry[16] << 21 + + carry[7] = (s7 + (1 << 20)) >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[9] = (s9 + (1 << 20)) >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[11] = (s11 + (1 << 20)) >> 21 + s12 += carry[11] + s11 -= carry[11] << 21 + carry[13] = (s13 + (1 << 20)) >> 21 + s14 += carry[13] + s13 -= carry[13] << 21 + carry[15] = (s15 + (1 << 20)) >> 21 + s16 += carry[15] + s15 -= carry[15] << 21 + + s5 += s17 * 666643 + s6 += s17 * 470296 + s7 += s17 * 654183 + s8 -= s17 * 997805 + s9 += s17 * 136657 + s10 -= s17 * 683901 + s17 = 0 + + s4 += s16 * 666643 + s5 += s16 * 470296 + s6 += s16 * 654183 + s7 -= s16 * 997805 + s8 += s16 * 136657 + s9 -= s16 * 683901 + s16 = 0 + + s3 += s15 * 666643 + s4 += s15 * 470296 + s5 += s15 * 654183 + s6 -= s15 * 997805 + s7 += s15 * 136657 + s8 -= s15 * 683901 + s15 = 0 + + s2 += s14 * 666643 + s3 += s14 * 470296 + s4 += s14 * 654183 + s5 -= s14 * 997805 + s6 += s14 * 136657 + s7 -= s14 * 683901 + s14 = 0 + + s1 += s13 * 666643 + s2 += s13 * 470296 + s3 += s13 * 654183 + s4 -= s13 * 997805 + s5 += s13 * 136657 + s6 -= s13 * 683901 + s13 = 0 + + s0 += s12 * 666643 + s1 += s12 * 470296 + s2 += s12 * 654183 + s3 -= s12 * 997805 + s4 += s12 * 136657 + s5 -= s12 * 683901 + s12 = 0 + + carry[0] = (s0 + (1 << 20)) >> 21 + s1 += carry[0] + s0 -= carry[0] << 21 + carry[2] = (s2 + (1 << 20)) >> 21 + s3 += carry[2] + s2 -= carry[2] << 21 + carry[4] = (s4 + (1 << 20)) >> 21 + s5 += carry[4] + s4 -= carry[4] << 21 + carry[6] = (s6 + (1 << 20)) >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[8] = (s8 + (1 << 20)) >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[10] = (s10 + (1 << 20)) >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + + carry[1] = (s1 + (1 << 20)) >> 21 + s2 += carry[1] + s1 -= carry[1] << 21 + carry[3] = (s3 + (1 << 20)) >> 21 + s4 += carry[3] + s3 -= carry[3] << 21 + carry[5] = (s5 + (1 << 20)) >> 21 + s6 += carry[5] + s5 -= carry[5] << 21 + carry[7] = (s7 + (1 << 20)) >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[9] = (s9 + (1 << 20)) >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[11] = (s11 + (1 << 20)) >> 21 + s12 += carry[11] + s11 -= carry[11] << 21 + + s0 += s12 * 666643 + s1 += s12 * 470296 + s2 += s12 * 654183 + s3 -= s12 * 997805 + s4 += s12 * 136657 + s5 -= s12 * 683901 + s12 = 0 + + carry[0] = s0 >> 21 + s1 += carry[0] + s0 -= carry[0] << 21 + carry[1] = s1 >> 21 + s2 += carry[1] + s1 -= carry[1] << 21 + carry[2] = s2 >> 21 + s3 += carry[2] + s2 -= carry[2] << 21 + carry[3] = s3 >> 21 + s4 += carry[3] + s3 -= carry[3] << 21 + carry[4] = s4 >> 21 + s5 += carry[4] + s4 -= carry[4] << 21 + carry[5] = s5 >> 21 + s6 += carry[5] + s5 -= carry[5] << 21 + carry[6] = s6 >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[7] = s7 >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[8] = s8 >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[9] = s9 >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[10] = s10 >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + carry[11] = s11 >> 21 + s12 += carry[11] + s11 -= carry[11] << 21 + + s0 += s12 * 666643 + s1 += s12 * 470296 + s2 += s12 * 654183 + s3 -= s12 * 997805 + s4 += s12 * 136657 + s5 -= s12 * 683901 + s12 = 0 + + carry[0] = s0 >> 21 + s1 += carry[0] + s0 -= carry[0] << 21 + carry[1] = s1 >> 21 + s2 += carry[1] + s1 -= carry[1] << 21 + carry[2] = s2 >> 21 + s3 += carry[2] + s2 -= carry[2] << 21 + carry[3] = s3 >> 21 + s4 += carry[3] + s3 -= carry[3] << 21 + carry[4] = s4 >> 21 + s5 += carry[4] + s4 -= carry[4] << 21 + carry[5] = s5 >> 21 + s6 += carry[5] + s5 -= carry[5] << 21 + carry[6] = s6 >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[7] = s7 >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[8] = s8 >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[9] = s9 >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[10] = s10 >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + + s[0] = byte(s0 >> 0) + s[1] = byte(s0 >> 8) + s[2] = byte((s0 >> 16) | (s1 << 5)) + s[3] = byte(s1 >> 3) + s[4] = byte(s1 >> 11) + s[5] = byte((s1 >> 19) | (s2 << 2)) + s[6] = byte(s2 >> 6) + s[7] = byte((s2 >> 14) | (s3 << 7)) + s[8] = byte(s3 >> 1) + s[9] = byte(s3 >> 9) + s[10] = byte((s3 >> 17) | (s4 << 4)) + s[11] = byte(s4 >> 4) + s[12] = byte(s4 >> 12) + s[13] = byte((s4 >> 20) | (s5 << 1)) + s[14] = byte(s5 >> 7) + s[15] = byte((s5 >> 15) | (s6 << 6)) + s[16] = byte(s6 >> 2) + s[17] = byte(s6 >> 10) + s[18] = byte((s6 >> 18) | (s7 << 3)) + s[19] = byte(s7 >> 5) + s[20] = byte(s7 >> 13) + s[21] = byte(s8 >> 0) + s[22] = byte(s8 >> 8) + s[23] = byte((s8 >> 16) | (s9 << 5)) + s[24] = byte(s9 >> 3) + s[25] = byte(s9 >> 11) + s[26] = byte((s9 >> 19) | (s10 << 2)) + s[27] = byte(s10 >> 6) + s[28] = byte((s10 >> 14) | (s11 << 7)) + s[29] = byte(s11 >> 1) + s[30] = byte(s11 >> 9) + s[31] = byte(s11 >> 17) +} + +// Input: +// s[0]+256*s[1]+...+256^63*s[63] = s +// +// Output: +// s[0]+256*s[1]+...+256^31*s[31] = s mod l +// where l = 2^252 + 27742317777372353535851937790883648493. +func ScReduce(out *[32]byte, s *[64]byte) { + s0 := 2097151 & load3(s[:]) + s1 := 2097151 & (load4(s[2:]) >> 5) + s2 := 2097151 & (load3(s[5:]) >> 2) + s3 := 2097151 & (load4(s[7:]) >> 7) + s4 := 2097151 & (load4(s[10:]) >> 4) + s5 := 2097151 & (load3(s[13:]) >> 1) + s6 := 2097151 & (load4(s[15:]) >> 6) + s7 := 2097151 & (load3(s[18:]) >> 3) + s8 := 2097151 & load3(s[21:]) + s9 := 2097151 & (load4(s[23:]) >> 5) + s10 := 2097151 & (load3(s[26:]) >> 2) + s11 := 2097151 & (load4(s[28:]) >> 7) + s12 := 2097151 & (load4(s[31:]) >> 4) + s13 := 2097151 & (load3(s[34:]) >> 1) + s14 := 2097151 & (load4(s[36:]) >> 6) + s15 := 2097151 & (load3(s[39:]) >> 3) + s16 := 2097151 & load3(s[42:]) + s17 := 2097151 & (load4(s[44:]) >> 5) + s18 := 2097151 & (load3(s[47:]) >> 2) + s19 := 2097151 & (load4(s[49:]) >> 7) + s20 := 2097151 & (load4(s[52:]) >> 4) + s21 := 2097151 & (load3(s[55:]) >> 1) + s22 := 2097151 & (load4(s[57:]) >> 6) + s23 := (load4(s[60:]) >> 3) + + s11 += s23 * 666643 + s12 += s23 * 470296 + s13 += s23 * 654183 + s14 -= s23 * 997805 + s15 += s23 * 136657 + s16 -= s23 * 683901 + s23 = 0 + + s10 += s22 * 666643 + s11 += s22 * 470296 + s12 += s22 * 654183 + s13 -= s22 * 997805 + s14 += s22 * 136657 + s15 -= s22 * 683901 + s22 = 0 + + s9 += s21 * 666643 + s10 += s21 * 470296 + s11 += s21 * 654183 + s12 -= s21 * 997805 + s13 += s21 * 136657 + s14 -= s21 * 683901 + s21 = 0 + + s8 += s20 * 666643 + s9 += s20 * 470296 + s10 += s20 * 654183 + s11 -= s20 * 997805 + s12 += s20 * 136657 + s13 -= s20 * 683901 + s20 = 0 + + s7 += s19 * 666643 + s8 += s19 * 470296 + s9 += s19 * 654183 + s10 -= s19 * 997805 + s11 += s19 * 136657 + s12 -= s19 * 683901 + s19 = 0 + + s6 += s18 * 666643 + s7 += s18 * 470296 + s8 += s18 * 654183 + s9 -= s18 * 997805 + s10 += s18 * 136657 + s11 -= s18 * 683901 + s18 = 0 + + var carry [17]int64 + + carry[6] = (s6 + (1 << 20)) >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[8] = (s8 + (1 << 20)) >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[10] = (s10 + (1 << 20)) >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + carry[12] = (s12 + (1 << 20)) >> 21 + s13 += carry[12] + s12 -= carry[12] << 21 + carry[14] = (s14 + (1 << 20)) >> 21 + s15 += carry[14] + s14 -= carry[14] << 21 + carry[16] = (s16 + (1 << 20)) >> 21 + s17 += carry[16] + s16 -= carry[16] << 21 + + carry[7] = (s7 + (1 << 20)) >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[9] = (s9 + (1 << 20)) >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[11] = (s11 + (1 << 20)) >> 21 + s12 += carry[11] + s11 -= carry[11] << 21 + carry[13] = (s13 + (1 << 20)) >> 21 + s14 += carry[13] + s13 -= carry[13] << 21 + carry[15] = (s15 + (1 << 20)) >> 21 + s16 += carry[15] + s15 -= carry[15] << 21 + + s5 += s17 * 666643 + s6 += s17 * 470296 + s7 += s17 * 654183 + s8 -= s17 * 997805 + s9 += s17 * 136657 + s10 -= s17 * 683901 + s17 = 0 + + s4 += s16 * 666643 + s5 += s16 * 470296 + s6 += s16 * 654183 + s7 -= s16 * 997805 + s8 += s16 * 136657 + s9 -= s16 * 683901 + s16 = 0 + + s3 += s15 * 666643 + s4 += s15 * 470296 + s5 += s15 * 654183 + s6 -= s15 * 997805 + s7 += s15 * 136657 + s8 -= s15 * 683901 + s15 = 0 + + s2 += s14 * 666643 + s3 += s14 * 470296 + s4 += s14 * 654183 + s5 -= s14 * 997805 + s6 += s14 * 136657 + s7 -= s14 * 683901 + s14 = 0 + + s1 += s13 * 666643 + s2 += s13 * 470296 + s3 += s13 * 654183 + s4 -= s13 * 997805 + s5 += s13 * 136657 + s6 -= s13 * 683901 + s13 = 0 + + s0 += s12 * 666643 + s1 += s12 * 470296 + s2 += s12 * 654183 + s3 -= s12 * 997805 + s4 += s12 * 136657 + s5 -= s12 * 683901 + s12 = 0 + + carry[0] = (s0 + (1 << 20)) >> 21 + s1 += carry[0] + s0 -= carry[0] << 21 + carry[2] = (s2 + (1 << 20)) >> 21 + s3 += carry[2] + s2 -= carry[2] << 21 + carry[4] = (s4 + (1 << 20)) >> 21 + s5 += carry[4] + s4 -= carry[4] << 21 + carry[6] = (s6 + (1 << 20)) >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[8] = (s8 + (1 << 20)) >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[10] = (s10 + (1 << 20)) >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + + carry[1] = (s1 + (1 << 20)) >> 21 + s2 += carry[1] + s1 -= carry[1] << 21 + carry[3] = (s3 + (1 << 20)) >> 21 + s4 += carry[3] + s3 -= carry[3] << 21 + carry[5] = (s5 + (1 << 20)) >> 21 + s6 += carry[5] + s5 -= carry[5] << 21 + carry[7] = (s7 + (1 << 20)) >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[9] = (s9 + (1 << 20)) >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[11] = (s11 + (1 << 20)) >> 21 + s12 += carry[11] + s11 -= carry[11] << 21 + + s0 += s12 * 666643 + s1 += s12 * 470296 + s2 += s12 * 654183 + s3 -= s12 * 997805 + s4 += s12 * 136657 + s5 -= s12 * 683901 + s12 = 0 + + carry[0] = s0 >> 21 + s1 += carry[0] + s0 -= carry[0] << 21 + carry[1] = s1 >> 21 + s2 += carry[1] + s1 -= carry[1] << 21 + carry[2] = s2 >> 21 + s3 += carry[2] + s2 -= carry[2] << 21 + carry[3] = s3 >> 21 + s4 += carry[3] + s3 -= carry[3] << 21 + carry[4] = s4 >> 21 + s5 += carry[4] + s4 -= carry[4] << 21 + carry[5] = s5 >> 21 + s6 += carry[5] + s5 -= carry[5] << 21 + carry[6] = s6 >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[7] = s7 >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[8] = s8 >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[9] = s9 >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[10] = s10 >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + carry[11] = s11 >> 21 + s12 += carry[11] + s11 -= carry[11] << 21 + + s0 += s12 * 666643 + s1 += s12 * 470296 + s2 += s12 * 654183 + s3 -= s12 * 997805 + s4 += s12 * 136657 + s5 -= s12 * 683901 + s12 = 0 + + carry[0] = s0 >> 21 + s1 += carry[0] + s0 -= carry[0] << 21 + carry[1] = s1 >> 21 + s2 += carry[1] + s1 -= carry[1] << 21 + carry[2] = s2 >> 21 + s3 += carry[2] + s2 -= carry[2] << 21 + carry[3] = s3 >> 21 + s4 += carry[3] + s3 -= carry[3] << 21 + carry[4] = s4 >> 21 + s5 += carry[4] + s4 -= carry[4] << 21 + carry[5] = s5 >> 21 + s6 += carry[5] + s5 -= carry[5] << 21 + carry[6] = s6 >> 21 + s7 += carry[6] + s6 -= carry[6] << 21 + carry[7] = s7 >> 21 + s8 += carry[7] + s7 -= carry[7] << 21 + carry[8] = s8 >> 21 + s9 += carry[8] + s8 -= carry[8] << 21 + carry[9] = s9 >> 21 + s10 += carry[9] + s9 -= carry[9] << 21 + carry[10] = s10 >> 21 + s11 += carry[10] + s10 -= carry[10] << 21 + + out[0] = byte(s0 >> 0) + out[1] = byte(s0 >> 8) + out[2] = byte((s0 >> 16) | (s1 << 5)) + out[3] = byte(s1 >> 3) + out[4] = byte(s1 >> 11) + out[5] = byte((s1 >> 19) | (s2 << 2)) + out[6] = byte(s2 >> 6) + out[7] = byte((s2 >> 14) | (s3 << 7)) + out[8] = byte(s3 >> 1) + out[9] = byte(s3 >> 9) + out[10] = byte((s3 >> 17) | (s4 << 4)) + out[11] = byte(s4 >> 4) + out[12] = byte(s4 >> 12) + out[13] = byte((s4 >> 20) | (s5 << 1)) + out[14] = byte(s5 >> 7) + out[15] = byte((s5 >> 15) | (s6 << 6)) + out[16] = byte(s6 >> 2) + out[17] = byte(s6 >> 10) + out[18] = byte((s6 >> 18) | (s7 << 3)) + out[19] = byte(s7 >> 5) + out[20] = byte(s7 >> 13) + out[21] = byte(s8 >> 0) + out[22] = byte(s8 >> 8) + out[23] = byte((s8 >> 16) | (s9 << 5)) + out[24] = byte(s9 >> 3) + out[25] = byte(s9 >> 11) + out[26] = byte((s9 >> 19) | (s10 << 2)) + out[27] = byte(s10 >> 6) + out[28] = byte((s10 >> 14) | (s11 << 7)) + out[29] = byte(s11 >> 1) + out[30] = byte(s11 >> 9) + out[31] = byte(s11 >> 17) +} + +// order is the order of Curve25519 in little-endian form. +var order = [4]uint64{0x5812631a5cf5d3ed, 0x14def9dea2f79cd6, 0, 0x1000000000000000} + +// ScMinimal returns true if the given scalar is less than the order of the +// curve. +func ScMinimal(scalar *[32]byte) bool { + for i := 3; ; i-- { + v := binary.LittleEndian.Uint64(scalar[i*8:]) + if v > order[i] { + return false + } else if v < order[i] { + break + } else if i == 0 { + return false + } + } + + return true +} diff --git a/vendor/golang.org/x/crypto/ocsp/ocsp.go b/vendor/golang.org/x/crypto/ocsp/ocsp.go index 8ed8796f..5edc9c97 100644 --- a/vendor/golang.org/x/crypto/ocsp/ocsp.go +++ b/vendor/golang.org/x/crypto/ocsp/ocsp.go @@ -295,17 +295,17 @@ const ( // The enumerated reasons for revoking a certificate. See RFC 5280. const ( - Unspecified = iota - KeyCompromise = iota - CACompromise = iota - AffiliationChanged = iota - Superseded = iota - CessationOfOperation = iota - CertificateHold = iota - _ = iota - RemoveFromCRL = iota - PrivilegeWithdrawn = iota - AACompromise = iota + Unspecified = 0 + KeyCompromise = 1 + CACompromise = 2 + AffiliationChanged = 3 + Superseded = 4 + CessationOfOperation = 5 + CertificateHold = 6 + + RemoveFromCRL = 8 + PrivilegeWithdrawn = 9 + AACompromise = 10 ) // Request represents an OCSP request. See RFC 6960. @@ -450,8 +450,8 @@ func ParseRequest(bytes []byte) (*Request, error) { // then the signature over the response is checked. If issuer is not nil then // it will be used to validate the signature or embedded certificate. // -// Invalid signatures or parse failures will result in a ParseError. Error -// responses will result in a ResponseError. +// Invalid responses and parse failures will result in a ParseError. +// Error responses will result in a ResponseError. func ParseResponse(bytes []byte, issuer *x509.Certificate) (*Response, error) { return ParseResponseForCert(bytes, nil, issuer) } @@ -462,8 +462,8 @@ func ParseResponse(bytes []byte, issuer *x509.Certificate) (*Response, error) { // issuer is not nil then it will be used to validate the signature or embedded // certificate. // -// Invalid signatures or parse failures will result in a ParseError. Error -// responses will result in a ResponseError. +// Invalid responses and parse failures will result in a ParseError. +// Error responses will result in a ResponseError. func ParseResponseForCert(bytes []byte, cert, issuer *x509.Certificate) (*Response, error) { var resp responseASN1 rest, err := asn1.Unmarshal(bytes, &resp) @@ -488,18 +488,36 @@ func ParseResponseForCert(bytes []byte, cert, issuer *x509.Certificate) (*Respon return nil, err } - if len(basicResp.Certificates) > 1 { - return nil, ParseError("OCSP response contains bad number of certificates") - } - if n := len(basicResp.TBSResponseData.Responses); n == 0 || cert == nil && n > 1 { return nil, ParseError("OCSP response contains bad number of responses") } + var singleResp singleResponse + if cert == nil { + singleResp = basicResp.TBSResponseData.Responses[0] + } else { + match := false + for _, resp := range basicResp.TBSResponseData.Responses { + if cert.SerialNumber.Cmp(resp.CertID.SerialNumber) == 0 { + singleResp = resp + match = true + break + } + } + if !match { + return nil, ParseError("no response matching the supplied certificate") + } + } + ret := &Response{ TBSResponseData: basicResp.TBSResponseData.Raw, Signature: basicResp.Signature.RightAlign(), SignatureAlgorithm: getSignatureAlgorithmFromOID(basicResp.SignatureAlgorithm.Algorithm), + Extensions: singleResp.SingleExtensions, + SerialNumber: singleResp.CertID.SerialNumber, + ProducedAt: basicResp.TBSResponseData.ProducedAt, + ThisUpdate: singleResp.ThisUpdate, + NextUpdate: singleResp.NextUpdate, } // Handle the ResponderID CHOICE tag. ResponderID can be flattened into @@ -522,6 +540,13 @@ func ParseResponseForCert(bytes []byte, cert, issuer *x509.Certificate) (*Respon } if len(basicResp.Certificates) > 0 { + // Responders should only send a single certificate (if they + // send any) that connects the responder's certificate to the + // original issuer. We accept responses with multiple + // certificates due to a number responders sending them[1], but + // ignore all but the first. + // + // [1] https://github.com/golang/go/issues/21527 ret.Certificate, err = x509.ParseCertificate(basicResp.Certificates[0].FullBytes) if err != nil { return nil, err @@ -542,25 +567,14 @@ func ParseResponseForCert(bytes []byte, cert, issuer *x509.Certificate) (*Respon } } - var r singleResponse - for _, resp := range basicResp.TBSResponseData.Responses { - if cert == nil || cert.SerialNumber.Cmp(resp.CertID.SerialNumber) == 0 { - r = resp - break - } - } - - for _, ext := range r.SingleExtensions { + for _, ext := range singleResp.SingleExtensions { if ext.Critical { return nil, ParseError("unsupported critical extension") } } - ret.Extensions = r.SingleExtensions - - ret.SerialNumber = r.CertID.SerialNumber for h, oid := range hashOIDs { - if r.CertID.HashAlgorithm.Algorithm.Equal(oid) { + if singleResp.CertID.HashAlgorithm.Algorithm.Equal(oid) { ret.IssuerHash = h break } @@ -570,20 +584,16 @@ func ParseResponseForCert(bytes []byte, cert, issuer *x509.Certificate) (*Respon } switch { - case bool(r.Good): + case bool(singleResp.Good): ret.Status = Good - case bool(r.Unknown): + case bool(singleResp.Unknown): ret.Status = Unknown default: ret.Status = Revoked - ret.RevokedAt = r.Revoked.RevocationTime - ret.RevocationReason = int(r.Revoked.Reason) + ret.RevokedAt = singleResp.Revoked.RevocationTime + ret.RevocationReason = int(singleResp.Revoked.Reason) } - ret.ProducedAt = basicResp.TBSResponseData.ProducedAt - ret.ThisUpdate = r.ThisUpdate - ret.NextUpdate = r.NextUpdate - return ret, nil } @@ -652,7 +662,7 @@ func CreateRequest(cert, issuer *x509.Certificate, opts *RequestOptions) ([]byte // // The issuer cert is used to puplate the IssuerNameHash and IssuerKeyHash fields. // -// The template is used to populate the SerialNumber, RevocationStatus, RevokedAt, +// The template is used to populate the SerialNumber, Status, RevokedAt, // RevocationReason, ThisUpdate, and NextUpdate fields. // // If template.IssuerHash is not set, SHA1 will be used. @@ -753,7 +763,7 @@ func CreateResponse(issuer, responderCert *x509.Certificate, template Response, } if template.Certificate != nil { response.Certificates = []asn1.RawValue{ - asn1.RawValue{FullBytes: template.Certificate.Raw}, + {FullBytes: template.Certificate.Raw}, } } responseDER, err := asn1.Marshal(response) diff --git a/vendor/golang.org/x/net/bpf/asm.go b/vendor/golang.org/x/net/bpf/asm.go new file mode 100644 index 00000000..15e21b18 --- /dev/null +++ b/vendor/golang.org/x/net/bpf/asm.go @@ -0,0 +1,41 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bpf + +import "fmt" + +// Assemble converts insts into raw instructions suitable for loading +// into a BPF virtual machine. +// +// Currently, no optimization is attempted, the assembled program flow +// is exactly as provided. +func Assemble(insts []Instruction) ([]RawInstruction, error) { + ret := make([]RawInstruction, len(insts)) + var err error + for i, inst := range insts { + ret[i], err = inst.Assemble() + if err != nil { + return nil, fmt.Errorf("assembling instruction %d: %s", i+1, err) + } + } + return ret, nil +} + +// Disassemble attempts to parse raw back into +// Instructions. Unrecognized RawInstructions are assumed to be an +// extension not implemented by this package, and are passed through +// unchanged to the output. The allDecoded value reports whether insts +// contains no RawInstructions. +func Disassemble(raw []RawInstruction) (insts []Instruction, allDecoded bool) { + insts = make([]Instruction, len(raw)) + allDecoded = true + for i, r := range raw { + insts[i] = r.Disassemble() + if _, ok := insts[i].(RawInstruction); ok { + allDecoded = false + } + } + return insts, allDecoded +} diff --git a/vendor/golang.org/x/net/bpf/constants.go b/vendor/golang.org/x/net/bpf/constants.go new file mode 100644 index 00000000..b89ca352 --- /dev/null +++ b/vendor/golang.org/x/net/bpf/constants.go @@ -0,0 +1,218 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bpf + +// A Register is a register of the BPF virtual machine. +type Register uint16 + +const ( + // RegA is the accumulator register. RegA is always the + // destination register of ALU operations. + RegA Register = iota + // RegX is the indirection register, used by LoadIndirect + // operations. + RegX +) + +// An ALUOp is an arithmetic or logic operation. +type ALUOp uint16 + +// ALU binary operation types. +const ( + ALUOpAdd ALUOp = iota << 4 + ALUOpSub + ALUOpMul + ALUOpDiv + ALUOpOr + ALUOpAnd + ALUOpShiftLeft + ALUOpShiftRight + aluOpNeg // Not exported because it's the only unary ALU operation, and gets its own instruction type. + ALUOpMod + ALUOpXor +) + +// A JumpTest is a comparison operator used in conditional jumps. +type JumpTest uint16 + +// Supported operators for conditional jumps. +const ( + // K == A + JumpEqual JumpTest = iota + // K != A + JumpNotEqual + // K > A + JumpGreaterThan + // K < A + JumpLessThan + // K >= A + JumpGreaterOrEqual + // K <= A + JumpLessOrEqual + // K & A != 0 + JumpBitsSet + // K & A == 0 + JumpBitsNotSet +) + +// An Extension is a function call provided by the kernel that +// performs advanced operations that are expensive or impossible +// within the BPF virtual machine. +// +// Extensions are only implemented by the Linux kernel. +// +// TODO: should we prune this list? Some of these extensions seem +// either broken or near-impossible to use correctly, whereas other +// (len, random, ifindex) are quite useful. +type Extension int + +// Extension functions available in the Linux kernel. +const ( + // extOffset is the negative maximum number of instructions used + // to load instructions by overloading the K argument. + extOffset = -0x1000 + // ExtLen returns the length of the packet. + ExtLen Extension = 1 + // ExtProto returns the packet's L3 protocol type. + ExtProto Extension = 0 + // ExtType returns the packet's type (skb->pkt_type in the kernel) + // + // TODO: better documentation. How nice an API do we want to + // provide for these esoteric extensions? + ExtType Extension = 4 + // ExtPayloadOffset returns the offset of the packet payload, or + // the first protocol header that the kernel does not know how to + // parse. + ExtPayloadOffset Extension = 52 + // ExtInterfaceIndex returns the index of the interface on which + // the packet was received. + ExtInterfaceIndex Extension = 8 + // ExtNetlinkAttr returns the netlink attribute of type X at + // offset A. + ExtNetlinkAttr Extension = 12 + // ExtNetlinkAttrNested returns the nested netlink attribute of + // type X at offset A. + ExtNetlinkAttrNested Extension = 16 + // ExtMark returns the packet's mark value. + ExtMark Extension = 20 + // ExtQueue returns the packet's assigned hardware queue. + ExtQueue Extension = 24 + // ExtLinkLayerType returns the packet's hardware address type + // (e.g. Ethernet, Infiniband). + ExtLinkLayerType Extension = 28 + // ExtRXHash returns the packets receive hash. + // + // TODO: figure out what this rxhash actually is. + ExtRXHash Extension = 32 + // ExtCPUID returns the ID of the CPU processing the current + // packet. + ExtCPUID Extension = 36 + // ExtVLANTag returns the packet's VLAN tag. + ExtVLANTag Extension = 44 + // ExtVLANTagPresent returns non-zero if the packet has a VLAN + // tag. + // + // TODO: I think this might be a lie: it reads bit 0x1000 of the + // VLAN header, which changed meaning in recent revisions of the + // spec - this extension may now return meaningless information. + ExtVLANTagPresent Extension = 48 + // ExtVLANProto returns 0x8100 if the frame has a VLAN header, + // 0x88a8 if the frame has a "Q-in-Q" double VLAN header, or some + // other value if no VLAN information is present. + ExtVLANProto Extension = 60 + // ExtRand returns a uniformly random uint32. + ExtRand Extension = 56 +) + +// The following gives names to various bit patterns used in opcode construction. + +const ( + opMaskCls uint16 = 0x7 + // opClsLoad masks + opMaskLoadDest = 0x01 + opMaskLoadWidth = 0x18 + opMaskLoadMode = 0xe0 + // opClsALU + opMaskOperandSrc = 0x08 + opMaskOperator = 0xf0 + // opClsJump + opMaskJumpConst = 0x0f + opMaskJumpCond = 0xf0 +) + +const ( + // +---------------+-----------------+---+---+---+ + // | AddrMode (3b) | LoadWidth (2b) | 0 | 0 | 0 | + // +---------------+-----------------+---+---+---+ + opClsLoadA uint16 = iota + // +---------------+-----------------+---+---+---+ + // | AddrMode (3b) | LoadWidth (2b) | 0 | 0 | 1 | + // +---------------+-----------------+---+---+---+ + opClsLoadX + // +---+---+---+---+---+---+---+---+ + // | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | + // +---+---+---+---+---+---+---+---+ + opClsStoreA + // +---+---+---+---+---+---+---+---+ + // | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | + // +---+---+---+---+---+---+---+---+ + opClsStoreX + // +---------------+-----------------+---+---+---+ + // | Operator (4b) | OperandSrc (1b) | 1 | 0 | 0 | + // +---------------+-----------------+---+---+---+ + opClsALU + // +-----------------------------+---+---+---+---+ + // | TestOperator (4b) | 0 | 1 | 0 | 1 | + // +-----------------------------+---+---+---+---+ + opClsJump + // +---+-------------------------+---+---+---+---+ + // | 0 | 0 | 0 | RetSrc (1b) | 0 | 1 | 1 | 0 | + // +---+-------------------------+---+---+---+---+ + opClsReturn + // +---+-------------------------+---+---+---+---+ + // | 0 | 0 | 0 | TXAorTAX (1b) | 0 | 1 | 1 | 1 | + // +---+-------------------------+---+---+---+---+ + opClsMisc +) + +const ( + opAddrModeImmediate uint16 = iota << 5 + opAddrModeAbsolute + opAddrModeIndirect + opAddrModeScratch + opAddrModePacketLen // actually an extension, not an addressing mode. + opAddrModeMemShift +) + +const ( + opLoadWidth4 uint16 = iota << 3 + opLoadWidth2 + opLoadWidth1 +) + +// Operator defined by ALUOp* + +const ( + opALUSrcConstant uint16 = iota << 3 + opALUSrcX +) + +const ( + opJumpAlways = iota << 4 + opJumpEqual + opJumpGT + opJumpGE + opJumpSet +) + +const ( + opRetSrcConstant uint16 = iota << 4 + opRetSrcA +) + +const ( + opMiscTAX = 0x00 + opMiscTXA = 0x80 +) diff --git a/vendor/golang.org/x/net/bpf/doc.go b/vendor/golang.org/x/net/bpf/doc.go new file mode 100644 index 00000000..ae62feb5 --- /dev/null +++ b/vendor/golang.org/x/net/bpf/doc.go @@ -0,0 +1,82 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Package bpf implements marshaling and unmarshaling of programs for the +Berkeley Packet Filter virtual machine, and provides a Go implementation +of the virtual machine. + +BPF's main use is to specify a packet filter for network taps, so that +the kernel doesn't have to expensively copy every packet it sees to +userspace. However, it's been repurposed to other areas where running +user code in-kernel is needed. For example, Linux's seccomp uses BPF +to apply security policies to system calls. For simplicity, this +documentation refers only to packets, but other uses of BPF have their +own data payloads. + +BPF programs run in a restricted virtual machine. It has almost no +access to kernel functions, and while conditional branches are +allowed, they can only jump forwards, to guarantee that there are no +infinite loops. + +The virtual machine + +The BPF VM is an accumulator machine. Its main register, called +register A, is an implicit source and destination in all arithmetic +and logic operations. The machine also has 16 scratch registers for +temporary storage, and an indirection register (register X) for +indirect memory access. All registers are 32 bits wide. + +Each run of a BPF program is given one packet, which is placed in the +VM's read-only "main memory". LoadAbsolute and LoadIndirect +instructions can fetch up to 32 bits at a time into register A for +examination. + +The goal of a BPF program is to produce and return a verdict (uint32), +which tells the kernel what to do with the packet. In the context of +packet filtering, the returned value is the number of bytes of the +packet to forward to userspace, or 0 to ignore the packet. Other +contexts like seccomp define their own return values. + +In order to simplify programs, attempts to read past the end of the +packet terminate the program execution with a verdict of 0 (ignore +packet). This means that the vast majority of BPF programs don't need +to do any explicit bounds checking. + +In addition to the bytes of the packet, some BPF programs have access +to extensions, which are essentially calls to kernel utility +functions. Currently, the only extensions supported by this package +are the Linux packet filter extensions. + +Examples + +This packet filter selects all ARP packets. + + bpf.Assemble([]bpf.Instruction{ + // Load "EtherType" field from the ethernet header. + bpf.LoadAbsolute{Off: 12, Size: 2}, + // Skip over the next instruction if EtherType is not ARP. + bpf.JumpIf{Cond: bpf.JumpNotEqual, Val: 0x0806, SkipTrue: 1}, + // Verdict is "send up to 4k of the packet to userspace." + bpf.RetConstant{Val: 4096}, + // Verdict is "ignore packet." + bpf.RetConstant{Val: 0}, + }) + +This packet filter captures a random 1% sample of traffic. + + bpf.Assemble([]bpf.Instruction{ + // Get a 32-bit random number from the Linux kernel. + bpf.LoadExtension{Num: bpf.ExtRand}, + // 1% dice roll? + bpf.JumpIf{Cond: bpf.JumpLessThan, Val: 2^32/100, SkipFalse: 1}, + // Capture. + bpf.RetConstant{Val: 4096}, + // Ignore. + bpf.RetConstant{Val: 0}, + }) + +*/ +package bpf // import "golang.org/x/net/bpf" diff --git a/vendor/golang.org/x/net/bpf/instructions.go b/vendor/golang.org/x/net/bpf/instructions.go new file mode 100644 index 00000000..f9dc0e8e --- /dev/null +++ b/vendor/golang.org/x/net/bpf/instructions.go @@ -0,0 +1,704 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bpf + +import "fmt" + +// An Instruction is one instruction executed by the BPF virtual +// machine. +type Instruction interface { + // Assemble assembles the Instruction into a RawInstruction. + Assemble() (RawInstruction, error) +} + +// A RawInstruction is a raw BPF virtual machine instruction. +type RawInstruction struct { + // Operation to execute. + Op uint16 + // For conditional jump instructions, the number of instructions + // to skip if the condition is true/false. + Jt uint8 + Jf uint8 + // Constant parameter. The meaning depends on the Op. + K uint32 +} + +// Assemble implements the Instruction Assemble method. +func (ri RawInstruction) Assemble() (RawInstruction, error) { return ri, nil } + +// Disassemble parses ri into an Instruction and returns it. If ri is +// not recognized by this package, ri itself is returned. +func (ri RawInstruction) Disassemble() Instruction { + switch ri.Op & opMaskCls { + case opClsLoadA, opClsLoadX: + reg := Register(ri.Op & opMaskLoadDest) + sz := 0 + switch ri.Op & opMaskLoadWidth { + case opLoadWidth4: + sz = 4 + case opLoadWidth2: + sz = 2 + case opLoadWidth1: + sz = 1 + default: + return ri + } + switch ri.Op & opMaskLoadMode { + case opAddrModeImmediate: + if sz != 4 { + return ri + } + return LoadConstant{Dst: reg, Val: ri.K} + case opAddrModeScratch: + if sz != 4 || ri.K > 15 { + return ri + } + return LoadScratch{Dst: reg, N: int(ri.K)} + case opAddrModeAbsolute: + if ri.K > extOffset+0xffffffff { + return LoadExtension{Num: Extension(-extOffset + ri.K)} + } + return LoadAbsolute{Size: sz, Off: ri.K} + case opAddrModeIndirect: + return LoadIndirect{Size: sz, Off: ri.K} + case opAddrModePacketLen: + if sz != 4 { + return ri + } + return LoadExtension{Num: ExtLen} + case opAddrModeMemShift: + return LoadMemShift{Off: ri.K} + default: + return ri + } + + case opClsStoreA: + if ri.Op != opClsStoreA || ri.K > 15 { + return ri + } + return StoreScratch{Src: RegA, N: int(ri.K)} + + case opClsStoreX: + if ri.Op != opClsStoreX || ri.K > 15 { + return ri + } + return StoreScratch{Src: RegX, N: int(ri.K)} + + case opClsALU: + switch op := ALUOp(ri.Op & opMaskOperator); op { + case ALUOpAdd, ALUOpSub, ALUOpMul, ALUOpDiv, ALUOpOr, ALUOpAnd, ALUOpShiftLeft, ALUOpShiftRight, ALUOpMod, ALUOpXor: + if ri.Op&opMaskOperandSrc != 0 { + return ALUOpX{Op: op} + } + return ALUOpConstant{Op: op, Val: ri.K} + case aluOpNeg: + return NegateA{} + default: + return ri + } + + case opClsJump: + if ri.Op&opMaskJumpConst != opClsJump { + return ri + } + switch ri.Op & opMaskJumpCond { + case opJumpAlways: + return Jump{Skip: ri.K} + case opJumpEqual: + if ri.Jt == 0 { + return JumpIf{ + Cond: JumpNotEqual, + Val: ri.K, + SkipTrue: ri.Jf, + SkipFalse: 0, + } + } + return JumpIf{ + Cond: JumpEqual, + Val: ri.K, + SkipTrue: ri.Jt, + SkipFalse: ri.Jf, + } + case opJumpGT: + if ri.Jt == 0 { + return JumpIf{ + Cond: JumpLessOrEqual, + Val: ri.K, + SkipTrue: ri.Jf, + SkipFalse: 0, + } + } + return JumpIf{ + Cond: JumpGreaterThan, + Val: ri.K, + SkipTrue: ri.Jt, + SkipFalse: ri.Jf, + } + case opJumpGE: + if ri.Jt == 0 { + return JumpIf{ + Cond: JumpLessThan, + Val: ri.K, + SkipTrue: ri.Jf, + SkipFalse: 0, + } + } + return JumpIf{ + Cond: JumpGreaterOrEqual, + Val: ri.K, + SkipTrue: ri.Jt, + SkipFalse: ri.Jf, + } + case opJumpSet: + return JumpIf{ + Cond: JumpBitsSet, + Val: ri.K, + SkipTrue: ri.Jt, + SkipFalse: ri.Jf, + } + default: + return ri + } + + case opClsReturn: + switch ri.Op { + case opClsReturn | opRetSrcA: + return RetA{} + case opClsReturn | opRetSrcConstant: + return RetConstant{Val: ri.K} + default: + return ri + } + + case opClsMisc: + switch ri.Op { + case opClsMisc | opMiscTAX: + return TAX{} + case opClsMisc | opMiscTXA: + return TXA{} + default: + return ri + } + + default: + panic("unreachable") // switch is exhaustive on the bit pattern + } +} + +// LoadConstant loads Val into register Dst. +type LoadConstant struct { + Dst Register + Val uint32 +} + +// Assemble implements the Instruction Assemble method. +func (a LoadConstant) Assemble() (RawInstruction, error) { + return assembleLoad(a.Dst, 4, opAddrModeImmediate, a.Val) +} + +// String returns the instruction in assembler notation. +func (a LoadConstant) String() string { + switch a.Dst { + case RegA: + return fmt.Sprintf("ld #%d", a.Val) + case RegX: + return fmt.Sprintf("ldx #%d", a.Val) + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +// LoadScratch loads scratch[N] into register Dst. +type LoadScratch struct { + Dst Register + N int // 0-15 +} + +// Assemble implements the Instruction Assemble method. +func (a LoadScratch) Assemble() (RawInstruction, error) { + if a.N < 0 || a.N > 15 { + return RawInstruction{}, fmt.Errorf("invalid scratch slot %d", a.N) + } + return assembleLoad(a.Dst, 4, opAddrModeScratch, uint32(a.N)) +} + +// String returns the instruction in assembler notation. +func (a LoadScratch) String() string { + switch a.Dst { + case RegA: + return fmt.Sprintf("ld M[%d]", a.N) + case RegX: + return fmt.Sprintf("ldx M[%d]", a.N) + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +// LoadAbsolute loads packet[Off:Off+Size] as an integer value into +// register A. +type LoadAbsolute struct { + Off uint32 + Size int // 1, 2 or 4 +} + +// Assemble implements the Instruction Assemble method. +func (a LoadAbsolute) Assemble() (RawInstruction, error) { + return assembleLoad(RegA, a.Size, opAddrModeAbsolute, a.Off) +} + +// String returns the instruction in assembler notation. +func (a LoadAbsolute) String() string { + switch a.Size { + case 1: // byte + return fmt.Sprintf("ldb [%d]", a.Off) + case 2: // half word + return fmt.Sprintf("ldh [%d]", a.Off) + case 4: // word + if a.Off > extOffset+0xffffffff { + return LoadExtension{Num: Extension(a.Off + 0x1000)}.String() + } + return fmt.Sprintf("ld [%d]", a.Off) + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +// LoadIndirect loads packet[X+Off:X+Off+Size] as an integer value +// into register A. +type LoadIndirect struct { + Off uint32 + Size int // 1, 2 or 4 +} + +// Assemble implements the Instruction Assemble method. +func (a LoadIndirect) Assemble() (RawInstruction, error) { + return assembleLoad(RegA, a.Size, opAddrModeIndirect, a.Off) +} + +// String returns the instruction in assembler notation. +func (a LoadIndirect) String() string { + switch a.Size { + case 1: // byte + return fmt.Sprintf("ldb [x + %d]", a.Off) + case 2: // half word + return fmt.Sprintf("ldh [x + %d]", a.Off) + case 4: // word + return fmt.Sprintf("ld [x + %d]", a.Off) + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +// LoadMemShift multiplies the first 4 bits of the byte at packet[Off] +// by 4 and stores the result in register X. +// +// This instruction is mainly useful to load into X the length of an +// IPv4 packet header in a single instruction, rather than have to do +// the arithmetic on the header's first byte by hand. +type LoadMemShift struct { + Off uint32 +} + +// Assemble implements the Instruction Assemble method. +func (a LoadMemShift) Assemble() (RawInstruction, error) { + return assembleLoad(RegX, 1, opAddrModeMemShift, a.Off) +} + +// String returns the instruction in assembler notation. +func (a LoadMemShift) String() string { + return fmt.Sprintf("ldx 4*([%d]&0xf)", a.Off) +} + +// LoadExtension invokes a linux-specific extension and stores the +// result in register A. +type LoadExtension struct { + Num Extension +} + +// Assemble implements the Instruction Assemble method. +func (a LoadExtension) Assemble() (RawInstruction, error) { + if a.Num == ExtLen { + return assembleLoad(RegA, 4, opAddrModePacketLen, 0) + } + return assembleLoad(RegA, 4, opAddrModeAbsolute, uint32(extOffset+a.Num)) +} + +// String returns the instruction in assembler notation. +func (a LoadExtension) String() string { + switch a.Num { + case ExtLen: + return "ld #len" + case ExtProto: + return "ld #proto" + case ExtType: + return "ld #type" + case ExtPayloadOffset: + return "ld #poff" + case ExtInterfaceIndex: + return "ld #ifidx" + case ExtNetlinkAttr: + return "ld #nla" + case ExtNetlinkAttrNested: + return "ld #nlan" + case ExtMark: + return "ld #mark" + case ExtQueue: + return "ld #queue" + case ExtLinkLayerType: + return "ld #hatype" + case ExtRXHash: + return "ld #rxhash" + case ExtCPUID: + return "ld #cpu" + case ExtVLANTag: + return "ld #vlan_tci" + case ExtVLANTagPresent: + return "ld #vlan_avail" + case ExtVLANProto: + return "ld #vlan_tpid" + case ExtRand: + return "ld #rand" + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +// StoreScratch stores register Src into scratch[N]. +type StoreScratch struct { + Src Register + N int // 0-15 +} + +// Assemble implements the Instruction Assemble method. +func (a StoreScratch) Assemble() (RawInstruction, error) { + if a.N < 0 || a.N > 15 { + return RawInstruction{}, fmt.Errorf("invalid scratch slot %d", a.N) + } + var op uint16 + switch a.Src { + case RegA: + op = opClsStoreA + case RegX: + op = opClsStoreX + default: + return RawInstruction{}, fmt.Errorf("invalid source register %v", a.Src) + } + + return RawInstruction{ + Op: op, + K: uint32(a.N), + }, nil +} + +// String returns the instruction in assembler notation. +func (a StoreScratch) String() string { + switch a.Src { + case RegA: + return fmt.Sprintf("st M[%d]", a.N) + case RegX: + return fmt.Sprintf("stx M[%d]", a.N) + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +// ALUOpConstant executes A = A Val. +type ALUOpConstant struct { + Op ALUOp + Val uint32 +} + +// Assemble implements the Instruction Assemble method. +func (a ALUOpConstant) Assemble() (RawInstruction, error) { + return RawInstruction{ + Op: opClsALU | opALUSrcConstant | uint16(a.Op), + K: a.Val, + }, nil +} + +// String returns the instruction in assembler notation. +func (a ALUOpConstant) String() string { + switch a.Op { + case ALUOpAdd: + return fmt.Sprintf("add #%d", a.Val) + case ALUOpSub: + return fmt.Sprintf("sub #%d", a.Val) + case ALUOpMul: + return fmt.Sprintf("mul #%d", a.Val) + case ALUOpDiv: + return fmt.Sprintf("div #%d", a.Val) + case ALUOpMod: + return fmt.Sprintf("mod #%d", a.Val) + case ALUOpAnd: + return fmt.Sprintf("and #%d", a.Val) + case ALUOpOr: + return fmt.Sprintf("or #%d", a.Val) + case ALUOpXor: + return fmt.Sprintf("xor #%d", a.Val) + case ALUOpShiftLeft: + return fmt.Sprintf("lsh #%d", a.Val) + case ALUOpShiftRight: + return fmt.Sprintf("rsh #%d", a.Val) + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +// ALUOpX executes A = A X +type ALUOpX struct { + Op ALUOp +} + +// Assemble implements the Instruction Assemble method. +func (a ALUOpX) Assemble() (RawInstruction, error) { + return RawInstruction{ + Op: opClsALU | opALUSrcX | uint16(a.Op), + }, nil +} + +// String returns the instruction in assembler notation. +func (a ALUOpX) String() string { + switch a.Op { + case ALUOpAdd: + return "add x" + case ALUOpSub: + return "sub x" + case ALUOpMul: + return "mul x" + case ALUOpDiv: + return "div x" + case ALUOpMod: + return "mod x" + case ALUOpAnd: + return "and x" + case ALUOpOr: + return "or x" + case ALUOpXor: + return "xor x" + case ALUOpShiftLeft: + return "lsh x" + case ALUOpShiftRight: + return "rsh x" + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +// NegateA executes A = -A. +type NegateA struct{} + +// Assemble implements the Instruction Assemble method. +func (a NegateA) Assemble() (RawInstruction, error) { + return RawInstruction{ + Op: opClsALU | uint16(aluOpNeg), + }, nil +} + +// String returns the instruction in assembler notation. +func (a NegateA) String() string { + return fmt.Sprintf("neg") +} + +// Jump skips the following Skip instructions in the program. +type Jump struct { + Skip uint32 +} + +// Assemble implements the Instruction Assemble method. +func (a Jump) Assemble() (RawInstruction, error) { + return RawInstruction{ + Op: opClsJump | opJumpAlways, + K: a.Skip, + }, nil +} + +// String returns the instruction in assembler notation. +func (a Jump) String() string { + return fmt.Sprintf("ja %d", a.Skip) +} + +// JumpIf skips the following Skip instructions in the program if A +// Val is true. +type JumpIf struct { + Cond JumpTest + Val uint32 + SkipTrue uint8 + SkipFalse uint8 +} + +// Assemble implements the Instruction Assemble method. +func (a JumpIf) Assemble() (RawInstruction, error) { + var ( + cond uint16 + flip bool + ) + switch a.Cond { + case JumpEqual: + cond = opJumpEqual + case JumpNotEqual: + cond, flip = opJumpEqual, true + case JumpGreaterThan: + cond = opJumpGT + case JumpLessThan: + cond, flip = opJumpGE, true + case JumpGreaterOrEqual: + cond = opJumpGE + case JumpLessOrEqual: + cond, flip = opJumpGT, true + case JumpBitsSet: + cond = opJumpSet + case JumpBitsNotSet: + cond, flip = opJumpSet, true + default: + return RawInstruction{}, fmt.Errorf("unknown JumpTest %v", a.Cond) + } + jt, jf := a.SkipTrue, a.SkipFalse + if flip { + jt, jf = jf, jt + } + return RawInstruction{ + Op: opClsJump | cond, + Jt: jt, + Jf: jf, + K: a.Val, + }, nil +} + +// String returns the instruction in assembler notation. +func (a JumpIf) String() string { + switch a.Cond { + // K == A + case JumpEqual: + return conditionalJump(a, "jeq", "jneq") + // K != A + case JumpNotEqual: + return fmt.Sprintf("jneq #%d,%d", a.Val, a.SkipTrue) + // K > A + case JumpGreaterThan: + return conditionalJump(a, "jgt", "jle") + // K < A + case JumpLessThan: + return fmt.Sprintf("jlt #%d,%d", a.Val, a.SkipTrue) + // K >= A + case JumpGreaterOrEqual: + return conditionalJump(a, "jge", "jlt") + // K <= A + case JumpLessOrEqual: + return fmt.Sprintf("jle #%d,%d", a.Val, a.SkipTrue) + // K & A != 0 + case JumpBitsSet: + if a.SkipFalse > 0 { + return fmt.Sprintf("jset #%d,%d,%d", a.Val, a.SkipTrue, a.SkipFalse) + } + return fmt.Sprintf("jset #%d,%d", a.Val, a.SkipTrue) + // K & A == 0, there is no assembler instruction for JumpBitNotSet, use JumpBitSet and invert skips + case JumpBitsNotSet: + return JumpIf{Cond: JumpBitsSet, SkipTrue: a.SkipFalse, SkipFalse: a.SkipTrue, Val: a.Val}.String() + default: + return fmt.Sprintf("unknown instruction: %#v", a) + } +} + +func conditionalJump(inst JumpIf, positiveJump, negativeJump string) string { + if inst.SkipTrue > 0 { + if inst.SkipFalse > 0 { + return fmt.Sprintf("%s #%d,%d,%d", positiveJump, inst.Val, inst.SkipTrue, inst.SkipFalse) + } + return fmt.Sprintf("%s #%d,%d", positiveJump, inst.Val, inst.SkipTrue) + } + return fmt.Sprintf("%s #%d,%d", negativeJump, inst.Val, inst.SkipFalse) +} + +// RetA exits the BPF program, returning the value of register A. +type RetA struct{} + +// Assemble implements the Instruction Assemble method. +func (a RetA) Assemble() (RawInstruction, error) { + return RawInstruction{ + Op: opClsReturn | opRetSrcA, + }, nil +} + +// String returns the instruction in assembler notation. +func (a RetA) String() string { + return fmt.Sprintf("ret a") +} + +// RetConstant exits the BPF program, returning a constant value. +type RetConstant struct { + Val uint32 +} + +// Assemble implements the Instruction Assemble method. +func (a RetConstant) Assemble() (RawInstruction, error) { + return RawInstruction{ + Op: opClsReturn | opRetSrcConstant, + K: a.Val, + }, nil +} + +// String returns the instruction in assembler notation. +func (a RetConstant) String() string { + return fmt.Sprintf("ret #%d", a.Val) +} + +// TXA copies the value of register X to register A. +type TXA struct{} + +// Assemble implements the Instruction Assemble method. +func (a TXA) Assemble() (RawInstruction, error) { + return RawInstruction{ + Op: opClsMisc | opMiscTXA, + }, nil +} + +// String returns the instruction in assembler notation. +func (a TXA) String() string { + return fmt.Sprintf("txa") +} + +// TAX copies the value of register A to register X. +type TAX struct{} + +// Assemble implements the Instruction Assemble method. +func (a TAX) Assemble() (RawInstruction, error) { + return RawInstruction{ + Op: opClsMisc | opMiscTAX, + }, nil +} + +// String returns the instruction in assembler notation. +func (a TAX) String() string { + return fmt.Sprintf("tax") +} + +func assembleLoad(dst Register, loadSize int, mode uint16, k uint32) (RawInstruction, error) { + var ( + cls uint16 + sz uint16 + ) + switch dst { + case RegA: + cls = opClsLoadA + case RegX: + cls = opClsLoadX + default: + return RawInstruction{}, fmt.Errorf("invalid target register %v", dst) + } + switch loadSize { + case 1: + sz = opLoadWidth1 + case 2: + sz = opLoadWidth2 + case 4: + sz = opLoadWidth4 + default: + return RawInstruction{}, fmt.Errorf("invalid load byte length %d", sz) + } + return RawInstruction{ + Op: cls | sz | mode, + K: k, + }, nil +} diff --git a/vendor/golang.org/x/net/bpf/setter.go b/vendor/golang.org/x/net/bpf/setter.go new file mode 100644 index 00000000..43e35f0a --- /dev/null +++ b/vendor/golang.org/x/net/bpf/setter.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bpf + +// A Setter is a type which can attach a compiled BPF filter to itself. +type Setter interface { + SetBPF(filter []RawInstruction) error +} diff --git a/vendor/golang.org/x/net/bpf/vm.go b/vendor/golang.org/x/net/bpf/vm.go new file mode 100644 index 00000000..4c656f1e --- /dev/null +++ b/vendor/golang.org/x/net/bpf/vm.go @@ -0,0 +1,140 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bpf + +import ( + "errors" + "fmt" +) + +// A VM is an emulated BPF virtual machine. +type VM struct { + filter []Instruction +} + +// NewVM returns a new VM using the input BPF program. +func NewVM(filter []Instruction) (*VM, error) { + if len(filter) == 0 { + return nil, errors.New("one or more Instructions must be specified") + } + + for i, ins := range filter { + check := len(filter) - (i + 1) + switch ins := ins.(type) { + // Check for out-of-bounds jumps in instructions + case Jump: + if check <= int(ins.Skip) { + return nil, fmt.Errorf("cannot jump %d instructions; jumping past program bounds", ins.Skip) + } + case JumpIf: + if check <= int(ins.SkipTrue) { + return nil, fmt.Errorf("cannot jump %d instructions in true case; jumping past program bounds", ins.SkipTrue) + } + if check <= int(ins.SkipFalse) { + return nil, fmt.Errorf("cannot jump %d instructions in false case; jumping past program bounds", ins.SkipFalse) + } + // Check for division or modulus by zero + case ALUOpConstant: + if ins.Val != 0 { + break + } + + switch ins.Op { + case ALUOpDiv, ALUOpMod: + return nil, errors.New("cannot divide by zero using ALUOpConstant") + } + // Check for unknown extensions + case LoadExtension: + switch ins.Num { + case ExtLen: + default: + return nil, fmt.Errorf("extension %d not implemented", ins.Num) + } + } + } + + // Make sure last instruction is a return instruction + switch filter[len(filter)-1].(type) { + case RetA, RetConstant: + default: + return nil, errors.New("BPF program must end with RetA or RetConstant") + } + + // Though our VM works using disassembled instructions, we + // attempt to assemble the input filter anyway to ensure it is compatible + // with an operating system VM. + _, err := Assemble(filter) + + return &VM{ + filter: filter, + }, err +} + +// Run runs the VM's BPF program against the input bytes. +// Run returns the number of bytes accepted by the BPF program, and any errors +// which occurred while processing the program. +func (v *VM) Run(in []byte) (int, error) { + var ( + // Registers of the virtual machine + regA uint32 + regX uint32 + regScratch [16]uint32 + + // OK is true if the program should continue processing the next + // instruction, or false if not, causing the loop to break + ok = true + ) + + // TODO(mdlayher): implement: + // - NegateA: + // - would require a change from uint32 registers to int32 + // registers + + // TODO(mdlayher): add interop tests that check signedness of ALU + // operations against kernel implementation, and make sure Go + // implementation matches behavior + + for i := 0; i < len(v.filter) && ok; i++ { + ins := v.filter[i] + + switch ins := ins.(type) { + case ALUOpConstant: + regA = aluOpConstant(ins, regA) + case ALUOpX: + regA, ok = aluOpX(ins, regA, regX) + case Jump: + i += int(ins.Skip) + case JumpIf: + jump := jumpIf(ins, regA) + i += jump + case LoadAbsolute: + regA, ok = loadAbsolute(ins, in) + case LoadConstant: + regA, regX = loadConstant(ins, regA, regX) + case LoadExtension: + regA = loadExtension(ins, in) + case LoadIndirect: + regA, ok = loadIndirect(ins, in, regX) + case LoadMemShift: + regX, ok = loadMemShift(ins, in) + case LoadScratch: + regA, regX = loadScratch(ins, regScratch, regA, regX) + case RetA: + return int(regA), nil + case RetConstant: + return int(ins.Val), nil + case StoreScratch: + regScratch = storeScratch(ins, regScratch, regA, regX) + case TAX: + regX = regA + case TXA: + regA = regX + default: + return 0, fmt.Errorf("unknown Instruction at index %d: %T", i, ins) + } + } + + return 0, nil +} diff --git a/vendor/golang.org/x/net/bpf/vm_instructions.go b/vendor/golang.org/x/net/bpf/vm_instructions.go new file mode 100644 index 00000000..516f9462 --- /dev/null +++ b/vendor/golang.org/x/net/bpf/vm_instructions.go @@ -0,0 +1,174 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bpf + +import ( + "encoding/binary" + "fmt" +) + +func aluOpConstant(ins ALUOpConstant, regA uint32) uint32 { + return aluOpCommon(ins.Op, regA, ins.Val) +} + +func aluOpX(ins ALUOpX, regA uint32, regX uint32) (uint32, bool) { + // Guard against division or modulus by zero by terminating + // the program, as the OS BPF VM does + if regX == 0 { + switch ins.Op { + case ALUOpDiv, ALUOpMod: + return 0, false + } + } + + return aluOpCommon(ins.Op, regA, regX), true +} + +func aluOpCommon(op ALUOp, regA uint32, value uint32) uint32 { + switch op { + case ALUOpAdd: + return regA + value + case ALUOpSub: + return regA - value + case ALUOpMul: + return regA * value + case ALUOpDiv: + // Division by zero not permitted by NewVM and aluOpX checks + return regA / value + case ALUOpOr: + return regA | value + case ALUOpAnd: + return regA & value + case ALUOpShiftLeft: + return regA << value + case ALUOpShiftRight: + return regA >> value + case ALUOpMod: + // Modulus by zero not permitted by NewVM and aluOpX checks + return regA % value + case ALUOpXor: + return regA ^ value + default: + return regA + } +} + +func jumpIf(ins JumpIf, value uint32) int { + var ok bool + inV := uint32(ins.Val) + + switch ins.Cond { + case JumpEqual: + ok = value == inV + case JumpNotEqual: + ok = value != inV + case JumpGreaterThan: + ok = value > inV + case JumpLessThan: + ok = value < inV + case JumpGreaterOrEqual: + ok = value >= inV + case JumpLessOrEqual: + ok = value <= inV + case JumpBitsSet: + ok = (value & inV) != 0 + case JumpBitsNotSet: + ok = (value & inV) == 0 + } + + if ok { + return int(ins.SkipTrue) + } + + return int(ins.SkipFalse) +} + +func loadAbsolute(ins LoadAbsolute, in []byte) (uint32, bool) { + offset := int(ins.Off) + size := int(ins.Size) + + return loadCommon(in, offset, size) +} + +func loadConstant(ins LoadConstant, regA uint32, regX uint32) (uint32, uint32) { + switch ins.Dst { + case RegA: + regA = ins.Val + case RegX: + regX = ins.Val + } + + return regA, regX +} + +func loadExtension(ins LoadExtension, in []byte) uint32 { + switch ins.Num { + case ExtLen: + return uint32(len(in)) + default: + panic(fmt.Sprintf("unimplemented extension: %d", ins.Num)) + } +} + +func loadIndirect(ins LoadIndirect, in []byte, regX uint32) (uint32, bool) { + offset := int(ins.Off) + int(regX) + size := int(ins.Size) + + return loadCommon(in, offset, size) +} + +func loadMemShift(ins LoadMemShift, in []byte) (uint32, bool) { + offset := int(ins.Off) + + if !inBounds(len(in), offset, 0) { + return 0, false + } + + // Mask off high 4 bits and multiply low 4 bits by 4 + return uint32(in[offset]&0x0f) * 4, true +} + +func inBounds(inLen int, offset int, size int) bool { + return offset+size <= inLen +} + +func loadCommon(in []byte, offset int, size int) (uint32, bool) { + if !inBounds(len(in), offset, size) { + return 0, false + } + + switch size { + case 1: + return uint32(in[offset]), true + case 2: + return uint32(binary.BigEndian.Uint16(in[offset : offset+size])), true + case 4: + return uint32(binary.BigEndian.Uint32(in[offset : offset+size])), true + default: + panic(fmt.Sprintf("invalid load size: %d", size)) + } +} + +func loadScratch(ins LoadScratch, regScratch [16]uint32, regA uint32, regX uint32) (uint32, uint32) { + switch ins.Dst { + case RegA: + regA = regScratch[ins.N] + case RegX: + regX = regScratch[ins.N] + } + + return regA, regX +} + +func storeScratch(ins StoreScratch, regScratch [16]uint32, regA uint32, regX uint32) [16]uint32 { + switch ins.Src { + case RegA: + regScratch[ins.N] = regA + case RegX: + regScratch[ins.N] = regX + } + + return regScratch +} diff --git a/vendor/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go deleted file mode 100644 index f143ed6a..00000000 --- a/vendor/golang.org/x/net/context/context.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package context defines the Context type, which carries deadlines, -// cancelation signals, and other request-scoped values across API boundaries -// and between processes. -// -// Incoming requests to a server should create a Context, and outgoing calls to -// servers should accept a Context. The chain of function calls between must -// propagate the Context, optionally replacing it with a modified copy created -// using WithDeadline, WithTimeout, WithCancel, or WithValue. -// -// Programs that use Contexts should follow these rules to keep interfaces -// consistent across packages and enable static analysis tools to check context -// propagation: -// -// Do not store Contexts inside a struct type; instead, pass a Context -// explicitly to each function that needs it. The Context should be the first -// parameter, typically named ctx: -// -// func DoSomething(ctx context.Context, arg Arg) error { -// // ... use ctx ... -// } -// -// Do not pass a nil Context, even if a function permits it. Pass context.TODO -// if you are unsure about which Context to use. -// -// Use context Values only for request-scoped data that transits processes and -// APIs, not for passing optional parameters to functions. -// -// The same Context may be passed to functions running in different goroutines; -// Contexts are safe for simultaneous use by multiple goroutines. -// -// See http://blog.golang.org/context for example code for a server that uses -// Contexts. -package context // import "golang.org/x/net/context" - -import "time" - -// A Context carries a deadline, a cancelation signal, and other values across -// API boundaries. -// -// Context's methods may be called by multiple goroutines simultaneously. -type Context interface { - // Deadline returns the time when work done on behalf of this context - // should be canceled. Deadline returns ok==false when no deadline is - // set. Successive calls to Deadline return the same results. - Deadline() (deadline time.Time, ok bool) - - // Done returns a channel that's closed when work done on behalf of this - // context should be canceled. Done may return nil if this context can - // never be canceled. Successive calls to Done return the same value. - // - // WithCancel arranges for Done to be closed when cancel is called; - // WithDeadline arranges for Done to be closed when the deadline - // expires; WithTimeout arranges for Done to be closed when the timeout - // elapses. - // - // Done is provided for use in select statements: - // - // // Stream generates values with DoSomething and sends them to out - // // until DoSomething returns an error or ctx.Done is closed. - // func Stream(ctx context.Context, out chan<- Value) error { - // for { - // v, err := DoSomething(ctx) - // if err != nil { - // return err - // } - // select { - // case <-ctx.Done(): - // return ctx.Err() - // case out <- v: - // } - // } - // } - // - // See http://blog.golang.org/pipelines for more examples of how to use - // a Done channel for cancelation. - Done() <-chan struct{} - - // Err returns a non-nil error value after Done is closed. Err returns - // Canceled if the context was canceled or DeadlineExceeded if the - // context's deadline passed. No other values for Err are defined. - // After Done is closed, successive calls to Err return the same value. - Err() error - - // Value returns the value associated with this context for key, or nil - // if no value is associated with key. Successive calls to Value with - // the same key returns the same result. - // - // Use context values only for request-scoped data that transits - // processes and API boundaries, not for passing optional parameters to - // functions. - // - // A key identifies a specific value in a Context. Functions that wish - // to store values in Context typically allocate a key in a global - // variable then use that key as the argument to context.WithValue and - // Context.Value. A key can be any type that supports equality; - // packages should define keys as an unexported type to avoid - // collisions. - // - // Packages that define a Context key should provide type-safe accessors - // for the values stores using that key: - // - // // Package user defines a User type that's stored in Contexts. - // package user - // - // import "golang.org/x/net/context" - // - // // User is the type of value stored in the Contexts. - // type User struct {...} - // - // // key is an unexported type for keys defined in this package. - // // This prevents collisions with keys defined in other packages. - // type key int - // - // // userKey is the key for user.User values in Contexts. It is - // // unexported; clients use user.NewContext and user.FromContext - // // instead of using this key directly. - // var userKey key = 0 - // - // // NewContext returns a new Context that carries value u. - // func NewContext(ctx context.Context, u *User) context.Context { - // return context.WithValue(ctx, userKey, u) - // } - // - // // FromContext returns the User value stored in ctx, if any. - // func FromContext(ctx context.Context) (*User, bool) { - // u, ok := ctx.Value(userKey).(*User) - // return u, ok - // } - Value(key interface{}) interface{} -} - -// Background returns a non-nil, empty Context. It is never canceled, has no -// values, and has no deadline. It is typically used by the main function, -// initialization, and tests, and as the top-level Context for incoming -// requests. -func Background() Context { - return background -} - -// TODO returns a non-nil, empty Context. Code should use context.TODO when -// it's unclear which Context to use or it is not yet available (because the -// surrounding function has not yet been extended to accept a Context -// parameter). TODO is recognized by static analysis tools that determine -// whether Contexts are propagated correctly in a program. -func TODO() Context { - return todo -} - -// A CancelFunc tells an operation to abandon its work. -// A CancelFunc does not wait for the work to stop. -// After the first call, subsequent calls to a CancelFunc do nothing. -type CancelFunc func() diff --git a/vendor/golang.org/x/net/context/go17.go b/vendor/golang.org/x/net/context/go17.go deleted file mode 100644 index d20f52b7..00000000 --- a/vendor/golang.org/x/net/context/go17.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build go1.7 - -package context - -import ( - "context" // standard library's context, as of Go 1.7 - "time" -) - -var ( - todo = context.TODO() - background = context.Background() -) - -// Canceled is the error returned by Context.Err when the context is canceled. -var Canceled = context.Canceled - -// DeadlineExceeded is the error returned by Context.Err when the context's -// deadline passes. -var DeadlineExceeded = context.DeadlineExceeded - -// WithCancel returns a copy of parent with a new Done channel. The returned -// context's Done channel is closed when the returned cancel function is called -// or when the parent context's Done channel is closed, whichever happens first. -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete. -func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { - ctx, f := context.WithCancel(parent) - return ctx, CancelFunc(f) -} - -// WithDeadline returns a copy of the parent context with the deadline adjusted -// to be no later than d. If the parent's deadline is already earlier than d, -// WithDeadline(parent, d) is semantically equivalent to parent. The returned -// context's Done channel is closed when the deadline expires, when the returned -// cancel function is called, or when the parent context's Done channel is -// closed, whichever happens first. -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete. -func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { - ctx, f := context.WithDeadline(parent, deadline) - return ctx, CancelFunc(f) -} - -// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete: -// -// func slowOperationWithTimeout(ctx context.Context) (Result, error) { -// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) -// defer cancel() // releases resources if slowOperation completes before timeout elapses -// return slowOperation(ctx) -// } -func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { - return WithDeadline(parent, time.Now().Add(timeout)) -} - -// WithValue returns a copy of parent in which the value associated with key is -// val. -// -// Use context Values only for request-scoped data that transits processes and -// APIs, not for passing optional parameters to functions. -func WithValue(parent Context, key interface{}, val interface{}) Context { - return context.WithValue(parent, key, val) -} diff --git a/vendor/golang.org/x/net/context/pre_go17.go b/vendor/golang.org/x/net/context/pre_go17.go deleted file mode 100644 index 0f35592d..00000000 --- a/vendor/golang.org/x/net/context/pre_go17.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !go1.7 - -package context - -import ( - "errors" - "fmt" - "sync" - "time" -) - -// An emptyCtx is never canceled, has no values, and has no deadline. It is not -// struct{}, since vars of this type must have distinct addresses. -type emptyCtx int - -func (*emptyCtx) Deadline() (deadline time.Time, ok bool) { - return -} - -func (*emptyCtx) Done() <-chan struct{} { - return nil -} - -func (*emptyCtx) Err() error { - return nil -} - -func (*emptyCtx) Value(key interface{}) interface{} { - return nil -} - -func (e *emptyCtx) String() string { - switch e { - case background: - return "context.Background" - case todo: - return "context.TODO" - } - return "unknown empty Context" -} - -var ( - background = new(emptyCtx) - todo = new(emptyCtx) -) - -// Canceled is the error returned by Context.Err when the context is canceled. -var Canceled = errors.New("context canceled") - -// DeadlineExceeded is the error returned by Context.Err when the context's -// deadline passes. -var DeadlineExceeded = errors.New("context deadline exceeded") - -// WithCancel returns a copy of parent with a new Done channel. The returned -// context's Done channel is closed when the returned cancel function is called -// or when the parent context's Done channel is closed, whichever happens first. -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete. -func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { - c := newCancelCtx(parent) - propagateCancel(parent, c) - return c, func() { c.cancel(true, Canceled) } -} - -// newCancelCtx returns an initialized cancelCtx. -func newCancelCtx(parent Context) *cancelCtx { - return &cancelCtx{ - Context: parent, - done: make(chan struct{}), - } -} - -// propagateCancel arranges for child to be canceled when parent is. -func propagateCancel(parent Context, child canceler) { - if parent.Done() == nil { - return // parent is never canceled - } - if p, ok := parentCancelCtx(parent); ok { - p.mu.Lock() - if p.err != nil { - // parent has already been canceled - child.cancel(false, p.err) - } else { - if p.children == nil { - p.children = make(map[canceler]bool) - } - p.children[child] = true - } - p.mu.Unlock() - } else { - go func() { - select { - case <-parent.Done(): - child.cancel(false, parent.Err()) - case <-child.Done(): - } - }() - } -} - -// parentCancelCtx follows a chain of parent references until it finds a -// *cancelCtx. This function understands how each of the concrete types in this -// package represents its parent. -func parentCancelCtx(parent Context) (*cancelCtx, bool) { - for { - switch c := parent.(type) { - case *cancelCtx: - return c, true - case *timerCtx: - return c.cancelCtx, true - case *valueCtx: - parent = c.Context - default: - return nil, false - } - } -} - -// removeChild removes a context from its parent. -func removeChild(parent Context, child canceler) { - p, ok := parentCancelCtx(parent) - if !ok { - return - } - p.mu.Lock() - if p.children != nil { - delete(p.children, child) - } - p.mu.Unlock() -} - -// A canceler is a context type that can be canceled directly. The -// implementations are *cancelCtx and *timerCtx. -type canceler interface { - cancel(removeFromParent bool, err error) - Done() <-chan struct{} -} - -// A cancelCtx can be canceled. When canceled, it also cancels any children -// that implement canceler. -type cancelCtx struct { - Context - - done chan struct{} // closed by the first cancel call. - - mu sync.Mutex - children map[canceler]bool // set to nil by the first cancel call - err error // set to non-nil by the first cancel call -} - -func (c *cancelCtx) Done() <-chan struct{} { - return c.done -} - -func (c *cancelCtx) Err() error { - c.mu.Lock() - defer c.mu.Unlock() - return c.err -} - -func (c *cancelCtx) String() string { - return fmt.Sprintf("%v.WithCancel", c.Context) -} - -// cancel closes c.done, cancels each of c's children, and, if -// removeFromParent is true, removes c from its parent's children. -func (c *cancelCtx) cancel(removeFromParent bool, err error) { - if err == nil { - panic("context: internal error: missing cancel error") - } - c.mu.Lock() - if c.err != nil { - c.mu.Unlock() - return // already canceled - } - c.err = err - close(c.done) - for child := range c.children { - // NOTE: acquiring the child's lock while holding parent's lock. - child.cancel(false, err) - } - c.children = nil - c.mu.Unlock() - - if removeFromParent { - removeChild(c.Context, c) - } -} - -// WithDeadline returns a copy of the parent context with the deadline adjusted -// to be no later than d. If the parent's deadline is already earlier than d, -// WithDeadline(parent, d) is semantically equivalent to parent. The returned -// context's Done channel is closed when the deadline expires, when the returned -// cancel function is called, or when the parent context's Done channel is -// closed, whichever happens first. -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete. -func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { - if cur, ok := parent.Deadline(); ok && cur.Before(deadline) { - // The current deadline is already sooner than the new one. - return WithCancel(parent) - } - c := &timerCtx{ - cancelCtx: newCancelCtx(parent), - deadline: deadline, - } - propagateCancel(parent, c) - d := deadline.Sub(time.Now()) - if d <= 0 { - c.cancel(true, DeadlineExceeded) // deadline has already passed - return c, func() { c.cancel(true, Canceled) } - } - c.mu.Lock() - defer c.mu.Unlock() - if c.err == nil { - c.timer = time.AfterFunc(d, func() { - c.cancel(true, DeadlineExceeded) - }) - } - return c, func() { c.cancel(true, Canceled) } -} - -// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to -// implement Done and Err. It implements cancel by stopping its timer then -// delegating to cancelCtx.cancel. -type timerCtx struct { - *cancelCtx - timer *time.Timer // Under cancelCtx.mu. - - deadline time.Time -} - -func (c *timerCtx) Deadline() (deadline time.Time, ok bool) { - return c.deadline, true -} - -func (c *timerCtx) String() string { - return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now())) -} - -func (c *timerCtx) cancel(removeFromParent bool, err error) { - c.cancelCtx.cancel(false, err) - if removeFromParent { - // Remove this timerCtx from its parent cancelCtx's children. - removeChild(c.cancelCtx.Context, c) - } - c.mu.Lock() - if c.timer != nil { - c.timer.Stop() - c.timer = nil - } - c.mu.Unlock() -} - -// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). -// -// Canceling this context releases resources associated with it, so code should -// call cancel as soon as the operations running in this Context complete: -// -// func slowOperationWithTimeout(ctx context.Context) (Result, error) { -// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) -// defer cancel() // releases resources if slowOperation completes before timeout elapses -// return slowOperation(ctx) -// } -func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { - return WithDeadline(parent, time.Now().Add(timeout)) -} - -// WithValue returns a copy of parent in which the value associated with key is -// val. -// -// Use context Values only for request-scoped data that transits processes and -// APIs, not for passing optional parameters to functions. -func WithValue(parent Context, key interface{}, val interface{}) Context { - return &valueCtx{parent, key, val} -} - -// A valueCtx carries a key-value pair. It implements Value for that key and -// delegates all other calls to the embedded Context. -type valueCtx struct { - Context - key, val interface{} -} - -func (c *valueCtx) String() string { - return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val) -} - -func (c *valueCtx) Value(key interface{}) interface{} { - if c.key == key { - return c.val - } - return c.Context.Value(key) -} diff --git a/vendor/golang.org/x/net/html/atom/gen.go b/vendor/golang.org/x/net/html/atom/gen.go index 6bfa8660..5d052781 100644 --- a/vendor/golang.org/x/net/html/atom/gen.go +++ b/vendor/golang.org/x/net/html/atom/gen.go @@ -4,17 +4,17 @@ // +build ignore +//go:generate go run gen.go +//go:generate go run gen.go -test + package main -// This program generates table.go and table_test.go. -// Invoke as -// -// go run gen.go |gofmt >table.go -// go run gen.go -test |gofmt >table_test.go - import ( + "bytes" "flag" "fmt" + "go/format" + "io/ioutil" "math/rand" "os" "sort" @@ -42,6 +42,18 @@ func identifier(s string) string { var test = flag.Bool("test", false, "generate table_test.go") +func genFile(name string, buf *bytes.Buffer) { + b, err := format.Source(buf.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := ioutil.WriteFile(name, b, 0644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + func main() { flag.Parse() @@ -52,32 +64,31 @@ func main() { all = append(all, extra...) sort.Strings(all) - if *test { - fmt.Printf("// generated by go run gen.go -test; DO NOT EDIT\n\n") - fmt.Printf("package atom\n\n") - fmt.Printf("var testAtomList = []string{\n") - for _, s := range all { - fmt.Printf("\t%q,\n", s) - } - fmt.Printf("}\n") - return - } - // uniq - lists have dups - // compute max len too - maxLen := 0 w := 0 for _, s := range all { if w == 0 || all[w-1] != s { - if maxLen < len(s) { - maxLen = len(s) - } all[w] = s w++ } } all = all[:w] + if *test { + var buf bytes.Buffer + fmt.Fprintln(&buf, "// Code generated by go generate gen.go; DO NOT EDIT.\n") + fmt.Fprintln(&buf, "//go:generate go run gen.go -test\n") + fmt.Fprintln(&buf, "package atom\n") + fmt.Fprintln(&buf, "var testAtomList = []string{") + for _, s := range all { + fmt.Fprintf(&buf, "\t%q,\n", s) + } + fmt.Fprintln(&buf, "}") + + genFile("table_test.go", &buf) + return + } + // Find hash that minimizes table size. var best *table for i := 0; i < 1000000; i++ { @@ -163,36 +174,46 @@ func main() { atom[s] = uint32(off<<8 | len(s)) } + var buf bytes.Buffer // Generate the Go code. - fmt.Printf("// generated by go run gen.go; DO NOT EDIT\n\n") - fmt.Printf("package atom\n\nconst (\n") + fmt.Fprintln(&buf, "// Code generated by go generate gen.go; DO NOT EDIT.\n") + fmt.Fprintln(&buf, "//go:generate go run gen.go\n") + fmt.Fprintln(&buf, "package atom\n\nconst (") + + // compute max len + maxLen := 0 for _, s := range all { - fmt.Printf("\t%s Atom = %#x\n", identifier(s), atom[s]) + if maxLen < len(s) { + maxLen = len(s) + } + fmt.Fprintf(&buf, "\t%s Atom = %#x\n", identifier(s), atom[s]) } - fmt.Printf(")\n\n") + fmt.Fprintln(&buf, ")\n") - fmt.Printf("const hash0 = %#x\n\n", best.h0) - fmt.Printf("const maxAtomLen = %d\n\n", maxLen) + fmt.Fprintf(&buf, "const hash0 = %#x\n\n", best.h0) + fmt.Fprintf(&buf, "const maxAtomLen = %d\n\n", maxLen) - fmt.Printf("var table = [1<<%d]Atom{\n", best.k) + fmt.Fprintf(&buf, "var table = [1<<%d]Atom{\n", best.k) for i, s := range best.tab { if s == "" { continue } - fmt.Printf("\t%#x: %#x, // %s\n", i, atom[s], s) + fmt.Fprintf(&buf, "\t%#x: %#x, // %s\n", i, atom[s], s) } - fmt.Printf("}\n") + fmt.Fprintf(&buf, "}\n") datasize := (1 << best.k) * 4 - fmt.Printf("const atomText =\n") + fmt.Fprintln(&buf, "const atomText =") textsize := len(text) for len(text) > 60 { - fmt.Printf("\t%q +\n", text[:60]) + fmt.Fprintf(&buf, "\t%q +\n", text[:60]) text = text[60:] } - fmt.Printf("\t%q\n\n", text) + fmt.Fprintf(&buf, "\t%q\n\n", text) - fmt.Fprintf(os.Stderr, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize) + genFile("table.go", &buf) + + fmt.Fprintf(os.Stdout, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize) } type byLen []string @@ -285,8 +306,10 @@ func (t *table) push(i uint32, depth int) bool { // The lists of element names and attribute keys were taken from // https://html.spec.whatwg.org/multipage/indices.html#index -// as of the "HTML Living Standard - Last Updated 21 February 2015" version. +// as of the "HTML Living Standard - Last Updated 16 April 2018" version. +// "command", "keygen" and "menuitem" have been removed from the spec, +// but are kept here for backwards compatibility. var elements = []string{ "a", "abbr", @@ -349,6 +372,7 @@ var elements = []string{ "legend", "li", "link", + "main", "map", "mark", "menu", @@ -364,6 +388,7 @@ var elements = []string{ "output", "p", "param", + "picture", "pre", "progress", "q", @@ -375,6 +400,7 @@ var elements = []string{ "script", "section", "select", + "slot", "small", "source", "span", @@ -403,14 +429,21 @@ var elements = []string{ } // https://html.spec.whatwg.org/multipage/indices.html#attributes-3 - +// +// "challenge", "command", "contextmenu", "dropzone", "icon", "keytype", "mediagroup", +// "radiogroup", "spellcheck", "scoped", "seamless", "sortable" and "sorted" have been removed from the spec, +// but are kept here for backwards compatibility. var attributes = []string{ "abbr", "accept", "accept-charset", "accesskey", "action", + "allowfullscreen", + "allowpaymentrequest", + "allowusermedia", "alt", + "as", "async", "autocomplete", "autofocus", @@ -420,6 +453,7 @@ var attributes = []string{ "checked", "cite", "class", + "color", "cols", "colspan", "command", @@ -457,6 +491,8 @@ var attributes = []string{ "icon", "id", "inputmode", + "integrity", + "is", "ismap", "itemid", "itemprop", @@ -481,16 +517,20 @@ var attributes = []string{ "multiple", "muted", "name", + "nomodule", + "nonce", "novalidate", "open", "optimum", "pattern", "ping", "placeholder", + "playsinline", "poster", "preload", "radiogroup", "readonly", + "referrerpolicy", "rel", "required", "reversed", @@ -507,10 +547,13 @@ var attributes = []string{ "sizes", "sortable", "sorted", + "slot", "span", + "spellcheck", "src", "srcdoc", "srclang", + "srcset", "start", "step", "style", @@ -520,16 +563,22 @@ var attributes = []string{ "translate", "type", "typemustmatch", + "updateviacache", "usemap", "value", "width", + "workertype", "wrap", } +// "onautocomplete", "onautocompleteerror", "onmousewheel", +// "onshow" and "onsort" have been removed from the spec, +// but are kept here for backwards compatibility. var eventHandlers = []string{ "onabort", "onautocomplete", "onautocompleteerror", + "onauxclick", "onafterprint", "onbeforeprint", "onbeforeunload", @@ -541,11 +590,14 @@ var eventHandlers = []string{ "onclick", "onclose", "oncontextmenu", + "oncopy", "oncuechange", + "oncut", "ondblclick", "ondrag", "ondragend", "ondragenter", + "ondragexit", "ondragleave", "ondragover", "ondragstart", @@ -565,18 +617,24 @@ var eventHandlers = []string{ "onload", "onloadeddata", "onloadedmetadata", + "onloadend", "onloadstart", "onmessage", + "onmessageerror", "onmousedown", + "onmouseenter", + "onmouseleave", "onmousemove", "onmouseout", "onmouseover", "onmouseup", "onmousewheel", + "onwheel", "onoffline", "ononline", "onpagehide", "onpageshow", + "onpaste", "onpause", "onplay", "onplaying", @@ -585,7 +643,9 @@ var eventHandlers = []string{ "onratechange", "onreset", "onresize", + "onrejectionhandled", "onscroll", + "onsecuritypolicyviolation", "onseeked", "onseeking", "onselect", @@ -597,6 +657,7 @@ var eventHandlers = []string{ "onsuspend", "ontimeupdate", "ontoggle", + "onunhandledrejection", "onunload", "onvolumechange", "onwaiting", @@ -604,6 +665,7 @@ var eventHandlers = []string{ // extra are ad-hoc values not covered by any of the lists above. var extra = []string{ + "acronym", "align", "annotation", "annotation-xml", @@ -639,6 +701,8 @@ var extra = []string{ "plaintext", "prompt", "public", + "rb", + "rtc", "spacer", "strike", "svg", diff --git a/vendor/golang.org/x/net/html/atom/table.go b/vendor/golang.org/x/net/html/atom/table.go index 2605ba31..2a938864 100644 --- a/vendor/golang.org/x/net/html/atom/table.go +++ b/vendor/golang.org/x/net/html/atom/table.go @@ -1,713 +1,783 @@ -// generated by go run gen.go; DO NOT EDIT +// Code generated by go generate gen.go; DO NOT EDIT. + +//go:generate go run gen.go package atom const ( - A Atom = 0x1 - Abbr Atom = 0x4 - Accept Atom = 0x2106 - AcceptCharset Atom = 0x210e - Accesskey Atom = 0x3309 - Action Atom = 0x1f606 - Address Atom = 0x4f307 - Align Atom = 0x1105 - Alt Atom = 0x4503 - Annotation Atom = 0x1670a - AnnotationXml Atom = 0x1670e - Applet Atom = 0x2b306 - Area Atom = 0x2fa04 - Article Atom = 0x38807 - Aside Atom = 0x8305 - Async Atom = 0x7b05 - Audio Atom = 0xa605 - Autocomplete Atom = 0x1fc0c - Autofocus Atom = 0xb309 - Autoplay Atom = 0xce08 - B Atom = 0x101 - Base Atom = 0xd604 - Basefont Atom = 0xd608 - Bdi Atom = 0x1a03 - Bdo Atom = 0xe703 - Bgsound Atom = 0x11807 - Big Atom = 0x12403 - Blink Atom = 0x12705 - Blockquote Atom = 0x12c0a - Body Atom = 0x2f04 - Br Atom = 0x202 - Button Atom = 0x13606 - Canvas Atom = 0x7f06 - Caption Atom = 0x1bb07 - Center Atom = 0x5b506 - Challenge Atom = 0x21f09 - Charset Atom = 0x2807 - Checked Atom = 0x32807 - Cite Atom = 0x3c804 - Class Atom = 0x4de05 - Code Atom = 0x14904 - Col Atom = 0x15003 - Colgroup Atom = 0x15008 - Color Atom = 0x15d05 - Cols Atom = 0x16204 - Colspan Atom = 0x16207 - Command Atom = 0x17507 - Content Atom = 0x42307 - Contenteditable Atom = 0x4230f - Contextmenu Atom = 0x3310b - Controls Atom = 0x18808 - Coords Atom = 0x19406 - Crossorigin Atom = 0x19f0b - Data Atom = 0x44a04 - Datalist Atom = 0x44a08 - Datetime Atom = 0x23c08 - Dd Atom = 0x26702 - Default Atom = 0x8607 - Defer Atom = 0x14b05 - Del Atom = 0x3ef03 - Desc Atom = 0x4db04 - Details Atom = 0x4807 - Dfn Atom = 0x6103 - Dialog Atom = 0x1b06 - Dir Atom = 0x6903 - Dirname Atom = 0x6907 - Disabled Atom = 0x10c08 - Div Atom = 0x11303 - Dl Atom = 0x11e02 - Download Atom = 0x40008 - Draggable Atom = 0x17b09 - Dropzone Atom = 0x39108 - Dt Atom = 0x50902 - Em Atom = 0x6502 - Embed Atom = 0x6505 - Enctype Atom = 0x21107 - Face Atom = 0x5b304 - Fieldset Atom = 0x1b008 - Figcaption Atom = 0x1b80a - Figure Atom = 0x1cc06 - Font Atom = 0xda04 - Footer Atom = 0x8d06 - For Atom = 0x1d803 - ForeignObject Atom = 0x1d80d - Foreignobject Atom = 0x1e50d - Form Atom = 0x1f204 - Formaction Atom = 0x1f20a - Formenctype Atom = 0x20d0b - Formmethod Atom = 0x2280a - Formnovalidate Atom = 0x2320e - Formtarget Atom = 0x2470a - Frame Atom = 0x9a05 - Frameset Atom = 0x9a08 - H1 Atom = 0x26e02 - H2 Atom = 0x29402 - H3 Atom = 0x2a702 - H4 Atom = 0x2e902 - H5 Atom = 0x2f302 - H6 Atom = 0x50b02 - Head Atom = 0x2d504 - Header Atom = 0x2d506 - Headers Atom = 0x2d507 - Height Atom = 0x25106 - Hgroup Atom = 0x25906 - Hidden Atom = 0x26506 - High Atom = 0x26b04 - Hr Atom = 0x27002 - Href Atom = 0x27004 - Hreflang Atom = 0x27008 - Html Atom = 0x25504 - HttpEquiv Atom = 0x2780a - I Atom = 0x601 - Icon Atom = 0x42204 - Id Atom = 0x8502 - Iframe Atom = 0x29606 - Image Atom = 0x29c05 - Img Atom = 0x2a103 - Input Atom = 0x3e805 - Inputmode Atom = 0x3e809 - Ins Atom = 0x1a803 - Isindex Atom = 0x2a907 - Ismap Atom = 0x2b005 - Itemid Atom = 0x33c06 - Itemprop Atom = 0x3c908 - Itemref Atom = 0x5ad07 - Itemscope Atom = 0x2b909 - Itemtype Atom = 0x2c308 - Kbd Atom = 0x1903 - Keygen Atom = 0x3906 - Keytype Atom = 0x53707 - Kind Atom = 0x10904 - Label Atom = 0xf005 - Lang Atom = 0x27404 - Legend Atom = 0x18206 - Li Atom = 0x1202 - Link Atom = 0x12804 - List Atom = 0x44e04 - Listing Atom = 0x44e07 - Loop Atom = 0xf404 - Low Atom = 0x11f03 - Malignmark Atom = 0x100a - Manifest Atom = 0x5f108 - Map Atom = 0x2b203 - Mark Atom = 0x1604 - Marquee Atom = 0x2cb07 - Math Atom = 0x2d204 - Max Atom = 0x2e103 - Maxlength Atom = 0x2e109 - Media Atom = 0x6e05 - Mediagroup Atom = 0x6e0a - Menu Atom = 0x33804 - Menuitem Atom = 0x33808 - Meta Atom = 0x45d04 - Meter Atom = 0x24205 - Method Atom = 0x22c06 - Mglyph Atom = 0x2a206 - Mi Atom = 0x2eb02 - Min Atom = 0x2eb03 - Minlength Atom = 0x2eb09 - Mn Atom = 0x23502 - Mo Atom = 0x3ed02 - Ms Atom = 0x2bc02 - Mtext Atom = 0x2f505 - Multiple Atom = 0x30308 - Muted Atom = 0x30b05 - Name Atom = 0x6c04 - Nav Atom = 0x3e03 - Nobr Atom = 0x5704 - Noembed Atom = 0x6307 - Noframes Atom = 0x9808 - Noscript Atom = 0x3d208 - Novalidate Atom = 0x2360a - Object Atom = 0x1ec06 - Ol Atom = 0xc902 - Onabort Atom = 0x13a07 - Onafterprint Atom = 0x1c00c - Onautocomplete Atom = 0x1fa0e - Onautocompleteerror Atom = 0x1fa13 - Onbeforeprint Atom = 0x6040d - Onbeforeunload Atom = 0x4e70e - Onblur Atom = 0xaa06 - Oncancel Atom = 0xe908 - Oncanplay Atom = 0x28509 - Oncanplaythrough Atom = 0x28510 - Onchange Atom = 0x3a708 - Onclick Atom = 0x31007 - Onclose Atom = 0x31707 - Oncontextmenu Atom = 0x32f0d - Oncuechange Atom = 0x3420b - Ondblclick Atom = 0x34d0a - Ondrag Atom = 0x35706 - Ondragend Atom = 0x35709 - Ondragenter Atom = 0x3600b - Ondragleave Atom = 0x36b0b - Ondragover Atom = 0x3760a - Ondragstart Atom = 0x3800b - Ondrop Atom = 0x38f06 - Ondurationchange Atom = 0x39f10 - Onemptied Atom = 0x39609 - Onended Atom = 0x3af07 - Onerror Atom = 0x3b607 - Onfocus Atom = 0x3bd07 - Onhashchange Atom = 0x3da0c - Oninput Atom = 0x3e607 - Oninvalid Atom = 0x3f209 - Onkeydown Atom = 0x3fb09 - Onkeypress Atom = 0x4080a - Onkeyup Atom = 0x41807 - Onlanguagechange Atom = 0x43210 - Onload Atom = 0x44206 - Onloadeddata Atom = 0x4420c - Onloadedmetadata Atom = 0x45510 - Onloadstart Atom = 0x46b0b - Onmessage Atom = 0x47609 - Onmousedown Atom = 0x47f0b - Onmousemove Atom = 0x48a0b - Onmouseout Atom = 0x4950a - Onmouseover Atom = 0x4a20b - Onmouseup Atom = 0x4ad09 - Onmousewheel Atom = 0x4b60c - Onoffline Atom = 0x4c209 - Ononline Atom = 0x4cb08 - Onpagehide Atom = 0x4d30a - Onpageshow Atom = 0x4fe0a - Onpause Atom = 0x50d07 - Onplay Atom = 0x51706 - Onplaying Atom = 0x51709 - Onpopstate Atom = 0x5200a - Onprogress Atom = 0x52a0a - Onratechange Atom = 0x53e0c - Onreset Atom = 0x54a07 - Onresize Atom = 0x55108 - Onscroll Atom = 0x55f08 - Onseeked Atom = 0x56708 - Onseeking Atom = 0x56f09 - Onselect Atom = 0x57808 - Onshow Atom = 0x58206 - Onsort Atom = 0x58b06 - Onstalled Atom = 0x59509 - Onstorage Atom = 0x59e09 - Onsubmit Atom = 0x5a708 - Onsuspend Atom = 0x5bb09 - Ontimeupdate Atom = 0xdb0c - Ontoggle Atom = 0x5c408 - Onunload Atom = 0x5cc08 - Onvolumechange Atom = 0x5d40e - Onwaiting Atom = 0x5e209 - Open Atom = 0x3cf04 - Optgroup Atom = 0xf608 - Optimum Atom = 0x5eb07 - Option Atom = 0x60006 - Output Atom = 0x49c06 - P Atom = 0xc01 - Param Atom = 0xc05 - Pattern Atom = 0x5107 - Ping Atom = 0x7704 - Placeholder Atom = 0xc30b - Plaintext Atom = 0xfd09 - Poster Atom = 0x15706 - Pre Atom = 0x25e03 - Preload Atom = 0x25e07 - Progress Atom = 0x52c08 - Prompt Atom = 0x5fa06 - Public Atom = 0x41e06 - Q Atom = 0x13101 - Radiogroup Atom = 0x30a - Readonly Atom = 0x2fb08 - Rel Atom = 0x25f03 - Required Atom = 0x1d008 - Reversed Atom = 0x5a08 - Rows Atom = 0x9204 - Rowspan Atom = 0x9207 - Rp Atom = 0x1c602 - Rt Atom = 0x13f02 - Ruby Atom = 0xaf04 - S Atom = 0x2c01 - Samp Atom = 0x4e04 - Sandbox Atom = 0xbb07 - Scope Atom = 0x2bd05 - Scoped Atom = 0x2bd06 - Script Atom = 0x3d406 - Seamless Atom = 0x31c08 - Section Atom = 0x4e207 - Select Atom = 0x57a06 - Selected Atom = 0x57a08 - Shape Atom = 0x4f905 - Size Atom = 0x55504 - Sizes Atom = 0x55505 - Small Atom = 0x18f05 - Sortable Atom = 0x58d08 - Sorted Atom = 0x19906 - Source Atom = 0x1aa06 - Spacer Atom = 0x2db06 - Span Atom = 0x9504 - Spellcheck Atom = 0x3230a - Src Atom = 0x3c303 - Srcdoc Atom = 0x3c306 - Srclang Atom = 0x41107 - Start Atom = 0x38605 - Step Atom = 0x5f704 - Strike Atom = 0x53306 - Strong Atom = 0x55906 - Style Atom = 0x61105 - Sub Atom = 0x5a903 - Summary Atom = 0x61607 - Sup Atom = 0x61d03 - Svg Atom = 0x62003 - System Atom = 0x62306 - Tabindex Atom = 0x46308 - Table Atom = 0x42d05 - Target Atom = 0x24b06 - Tbody Atom = 0x2e05 - Td Atom = 0x4702 - Template Atom = 0x62608 - Textarea Atom = 0x2f608 - Tfoot Atom = 0x8c05 - Th Atom = 0x22e02 - Thead Atom = 0x2d405 - Time Atom = 0xdd04 - Title Atom = 0xa105 - Tr Atom = 0x10502 - Track Atom = 0x10505 - Translate Atom = 0x14009 - Tt Atom = 0x5302 - Type Atom = 0x21404 - Typemustmatch Atom = 0x2140d - U Atom = 0xb01 - Ul Atom = 0x8a02 - Usemap Atom = 0x51106 - Value Atom = 0x4005 - Var Atom = 0x11503 - Video Atom = 0x28105 - Wbr Atom = 0x12103 - Width Atom = 0x50705 - Wrap Atom = 0x58704 - Xmp Atom = 0xc103 + A Atom = 0x1 + Abbr Atom = 0x4 + Accept Atom = 0x1a06 + AcceptCharset Atom = 0x1a0e + Accesskey Atom = 0x2c09 + Acronym Atom = 0xaa07 + Action Atom = 0x27206 + Address Atom = 0x6f307 + Align Atom = 0xb105 + Allowfullscreen Atom = 0x2080f + Allowpaymentrequest Atom = 0xc113 + Allowusermedia Atom = 0xdd0e + Alt Atom = 0xf303 + Annotation Atom = 0x1c90a + AnnotationXml Atom = 0x1c90e + Applet Atom = 0x31906 + Area Atom = 0x35604 + Article Atom = 0x3fc07 + As Atom = 0x3c02 + Aside Atom = 0x10705 + Async Atom = 0xff05 + Audio Atom = 0x11505 + Autocomplete Atom = 0x2780c + Autofocus Atom = 0x12109 + Autoplay Atom = 0x13c08 + B Atom = 0x101 + Base Atom = 0x3b04 + Basefont Atom = 0x3b08 + Bdi Atom = 0xba03 + Bdo Atom = 0x14b03 + Bgsound Atom = 0x15e07 + Big Atom = 0x17003 + Blink Atom = 0x17305 + Blockquote Atom = 0x1870a + Body Atom = 0x2804 + Br Atom = 0x202 + Button Atom = 0x19106 + Canvas Atom = 0x10306 + Caption Atom = 0x23107 + Center Atom = 0x22006 + Challenge Atom = 0x29b09 + Charset Atom = 0x2107 + Checked Atom = 0x47907 + Cite Atom = 0x19c04 + Class Atom = 0x56405 + Code Atom = 0x5c504 + Col Atom = 0x1ab03 + Colgroup Atom = 0x1ab08 + Color Atom = 0x1bf05 + Cols Atom = 0x1c404 + Colspan Atom = 0x1c407 + Command Atom = 0x1d707 + Content Atom = 0x58b07 + Contenteditable Atom = 0x58b0f + Contextmenu Atom = 0x3800b + Controls Atom = 0x1de08 + Coords Atom = 0x1ea06 + Crossorigin Atom = 0x1fb0b + Data Atom = 0x4a504 + Datalist Atom = 0x4a508 + Datetime Atom = 0x2b808 + Dd Atom = 0x2d702 + Default Atom = 0x10a07 + Defer Atom = 0x5c705 + Del Atom = 0x45203 + Desc Atom = 0x56104 + Details Atom = 0x7207 + Dfn Atom = 0x8703 + Dialog Atom = 0xbb06 + Dir Atom = 0x9303 + Dirname Atom = 0x9307 + Disabled Atom = 0x16408 + Div Atom = 0x16b03 + Dl Atom = 0x5e602 + Download Atom = 0x46308 + Draggable Atom = 0x17a09 + Dropzone Atom = 0x40508 + Dt Atom = 0x64b02 + Em Atom = 0x6e02 + Embed Atom = 0x6e05 + Enctype Atom = 0x28d07 + Face Atom = 0x21e04 + Fieldset Atom = 0x22608 + Figcaption Atom = 0x22e0a + Figure Atom = 0x24806 + Font Atom = 0x3f04 + Footer Atom = 0xf606 + For Atom = 0x25403 + ForeignObject Atom = 0x2540d + Foreignobject Atom = 0x2610d + Form Atom = 0x26e04 + Formaction Atom = 0x26e0a + Formenctype Atom = 0x2890b + Formmethod Atom = 0x2a40a + Formnovalidate Atom = 0x2ae0e + Formtarget Atom = 0x2c00a + Frame Atom = 0x8b05 + Frameset Atom = 0x8b08 + H1 Atom = 0x15c02 + H2 Atom = 0x2de02 + H3 Atom = 0x30d02 + H4 Atom = 0x34502 + H5 Atom = 0x34f02 + H6 Atom = 0x64d02 + Head Atom = 0x33104 + Header Atom = 0x33106 + Headers Atom = 0x33107 + Height Atom = 0x5206 + Hgroup Atom = 0x2ca06 + Hidden Atom = 0x2d506 + High Atom = 0x2db04 + Hr Atom = 0x15702 + Href Atom = 0x2e004 + Hreflang Atom = 0x2e008 + Html Atom = 0x5604 + HttpEquiv Atom = 0x2e80a + I Atom = 0x601 + Icon Atom = 0x58a04 + Id Atom = 0x10902 + Iframe Atom = 0x2fc06 + Image Atom = 0x30205 + Img Atom = 0x30703 + Input Atom = 0x44b05 + Inputmode Atom = 0x44b09 + Ins Atom = 0x20403 + Integrity Atom = 0x23f09 + Is Atom = 0x16502 + Isindex Atom = 0x30f07 + Ismap Atom = 0x31605 + Itemid Atom = 0x38b06 + Itemprop Atom = 0x19d08 + Itemref Atom = 0x3cd07 + Itemscope Atom = 0x67109 + Itemtype Atom = 0x31f08 + Kbd Atom = 0xb903 + Keygen Atom = 0x3206 + Keytype Atom = 0xd607 + Kind Atom = 0x17704 + Label Atom = 0x5905 + Lang Atom = 0x2e404 + Legend Atom = 0x18106 + Li Atom = 0xb202 + Link Atom = 0x17404 + List Atom = 0x4a904 + Listing Atom = 0x4a907 + Loop Atom = 0x5d04 + Low Atom = 0xc303 + Main Atom = 0x1004 + Malignmark Atom = 0xb00a + Manifest Atom = 0x6d708 + Map Atom = 0x31803 + Mark Atom = 0xb604 + Marquee Atom = 0x32707 + Math Atom = 0x32e04 + Max Atom = 0x33d03 + Maxlength Atom = 0x33d09 + Media Atom = 0xe605 + Mediagroup Atom = 0xe60a + Menu Atom = 0x38704 + Menuitem Atom = 0x38708 + Meta Atom = 0x4b804 + Meter Atom = 0x9805 + Method Atom = 0x2a806 + Mglyph Atom = 0x30806 + Mi Atom = 0x34702 + Min Atom = 0x34703 + Minlength Atom = 0x34709 + Mn Atom = 0x2b102 + Mo Atom = 0xa402 + Ms Atom = 0x67402 + Mtext Atom = 0x35105 + Multiple Atom = 0x35f08 + Muted Atom = 0x36705 + Name Atom = 0x9604 + Nav Atom = 0x1303 + Nobr Atom = 0x3704 + Noembed Atom = 0x6c07 + Noframes Atom = 0x8908 + Nomodule Atom = 0xa208 + Nonce Atom = 0x1a605 + Noscript Atom = 0x21608 + Novalidate Atom = 0x2b20a + Object Atom = 0x26806 + Ol Atom = 0x13702 + Onabort Atom = 0x19507 + Onafterprint Atom = 0x2360c + Onautocomplete Atom = 0x2760e + Onautocompleteerror Atom = 0x27613 + Onauxclick Atom = 0x61f0a + Onbeforeprint Atom = 0x69e0d + Onbeforeunload Atom = 0x6e70e + Onblur Atom = 0x56d06 + Oncancel Atom = 0x11908 + Oncanplay Atom = 0x14d09 + Oncanplaythrough Atom = 0x14d10 + Onchange Atom = 0x41b08 + Onclick Atom = 0x2f507 + Onclose Atom = 0x36c07 + Oncontextmenu Atom = 0x37e0d + Oncopy Atom = 0x39106 + Oncuechange Atom = 0x3970b + Oncut Atom = 0x3a205 + Ondblclick Atom = 0x3a70a + Ondrag Atom = 0x3b106 + Ondragend Atom = 0x3b109 + Ondragenter Atom = 0x3ba0b + Ondragexit Atom = 0x3c50a + Ondragleave Atom = 0x3df0b + Ondragover Atom = 0x3ea0a + Ondragstart Atom = 0x3f40b + Ondrop Atom = 0x40306 + Ondurationchange Atom = 0x41310 + Onemptied Atom = 0x40a09 + Onended Atom = 0x42307 + Onerror Atom = 0x42a07 + Onfocus Atom = 0x43107 + Onhashchange Atom = 0x43d0c + Oninput Atom = 0x44907 + Oninvalid Atom = 0x45509 + Onkeydown Atom = 0x45e09 + Onkeypress Atom = 0x46b0a + Onkeyup Atom = 0x48007 + Onlanguagechange Atom = 0x48d10 + Onload Atom = 0x49d06 + Onloadeddata Atom = 0x49d0c + Onloadedmetadata Atom = 0x4b010 + Onloadend Atom = 0x4c609 + Onloadstart Atom = 0x4cf0b + Onmessage Atom = 0x4da09 + Onmessageerror Atom = 0x4da0e + Onmousedown Atom = 0x4e80b + Onmouseenter Atom = 0x4f30c + Onmouseleave Atom = 0x4ff0c + Onmousemove Atom = 0x50b0b + Onmouseout Atom = 0x5160a + Onmouseover Atom = 0x5230b + Onmouseup Atom = 0x52e09 + Onmousewheel Atom = 0x53c0c + Onoffline Atom = 0x54809 + Ononline Atom = 0x55108 + Onpagehide Atom = 0x5590a + Onpageshow Atom = 0x5730a + Onpaste Atom = 0x57f07 + Onpause Atom = 0x59a07 + Onplay Atom = 0x5a406 + Onplaying Atom = 0x5a409 + Onpopstate Atom = 0x5ad0a + Onprogress Atom = 0x5b70a + Onratechange Atom = 0x5cc0c + Onrejectionhandled Atom = 0x5d812 + Onreset Atom = 0x5ea07 + Onresize Atom = 0x5f108 + Onscroll Atom = 0x60008 + Onsecuritypolicyviolation Atom = 0x60819 + Onseeked Atom = 0x62908 + Onseeking Atom = 0x63109 + Onselect Atom = 0x63a08 + Onshow Atom = 0x64406 + Onsort Atom = 0x64f06 + Onstalled Atom = 0x65909 + Onstorage Atom = 0x66209 + Onsubmit Atom = 0x66b08 + Onsuspend Atom = 0x67b09 + Ontimeupdate Atom = 0x400c + Ontoggle Atom = 0x68408 + Onunhandledrejection Atom = 0x68c14 + Onunload Atom = 0x6ab08 + Onvolumechange Atom = 0x6b30e + Onwaiting Atom = 0x6c109 + Onwheel Atom = 0x6ca07 + Open Atom = 0x1a304 + Optgroup Atom = 0x5f08 + Optimum Atom = 0x6d107 + Option Atom = 0x6e306 + Output Atom = 0x51d06 + P Atom = 0xc01 + Param Atom = 0xc05 + Pattern Atom = 0x6607 + Picture Atom = 0x7b07 + Ping Atom = 0xef04 + Placeholder Atom = 0x1310b + Plaintext Atom = 0x1b209 + Playsinline Atom = 0x1400b + Poster Atom = 0x2cf06 + Pre Atom = 0x47003 + Preload Atom = 0x48607 + Progress Atom = 0x5b908 + Prompt Atom = 0x53606 + Public Atom = 0x58606 + Q Atom = 0xcf01 + Radiogroup Atom = 0x30a + Rb Atom = 0x3a02 + Readonly Atom = 0x35708 + Referrerpolicy Atom = 0x3d10e + Rel Atom = 0x48703 + Required Atom = 0x24c08 + Reversed Atom = 0x8008 + Rows Atom = 0x9c04 + Rowspan Atom = 0x9c07 + Rp Atom = 0x23c02 + Rt Atom = 0x19a02 + Rtc Atom = 0x19a03 + Ruby Atom = 0xfb04 + S Atom = 0x2501 + Samp Atom = 0x7804 + Sandbox Atom = 0x12907 + Scope Atom = 0x67505 + Scoped Atom = 0x67506 + Script Atom = 0x21806 + Seamless Atom = 0x37108 + Section Atom = 0x56807 + Select Atom = 0x63c06 + Selected Atom = 0x63c08 + Shape Atom = 0x1e505 + Size Atom = 0x5f504 + Sizes Atom = 0x5f505 + Slot Atom = 0x1ef04 + Small Atom = 0x20605 + Sortable Atom = 0x65108 + Sorted Atom = 0x33706 + Source Atom = 0x37806 + Spacer Atom = 0x43706 + Span Atom = 0x9f04 + Spellcheck Atom = 0x4740a + Src Atom = 0x5c003 + Srcdoc Atom = 0x5c006 + Srclang Atom = 0x5f907 + Srcset Atom = 0x6f906 + Start Atom = 0x3fa05 + Step Atom = 0x58304 + Strike Atom = 0xd206 + Strong Atom = 0x6dd06 + Style Atom = 0x6ff05 + Sub Atom = 0x66d03 + Summary Atom = 0x70407 + Sup Atom = 0x70b03 + Svg Atom = 0x70e03 + System Atom = 0x71106 + Tabindex Atom = 0x4be08 + Table Atom = 0x59505 + Target Atom = 0x2c406 + Tbody Atom = 0x2705 + Td Atom = 0x9202 + Template Atom = 0x71408 + Textarea Atom = 0x35208 + Tfoot Atom = 0xf505 + Th Atom = 0x15602 + Thead Atom = 0x33005 + Time Atom = 0x4204 + Title Atom = 0x11005 + Tr Atom = 0xcc02 + Track Atom = 0x1ba05 + Translate Atom = 0x1f209 + Tt Atom = 0x6802 + Type Atom = 0xd904 + Typemustmatch Atom = 0x2900d + U Atom = 0xb01 + Ul Atom = 0xa702 + Updateviacache Atom = 0x460e + Usemap Atom = 0x59e06 + Value Atom = 0x1505 + Var Atom = 0x16d03 + Video Atom = 0x2f105 + Wbr Atom = 0x57c03 + Width Atom = 0x64905 + Workertype Atom = 0x71c0a + Wrap Atom = 0x72604 + Xmp Atom = 0x12f03 ) -const hash0 = 0xc17da63e +const hash0 = 0x81cdf10e -const maxAtomLen = 19 +const maxAtomLen = 25 var table = [1 << 9]Atom{ - 0x1: 0x48a0b, // onmousemove - 0x2: 0x5e209, // onwaiting - 0x3: 0x1fa13, // onautocompleteerror - 0x4: 0x5fa06, // prompt - 0x7: 0x5eb07, // optimum - 0x8: 0x1604, // mark - 0xa: 0x5ad07, // itemref - 0xb: 0x4fe0a, // onpageshow - 0xc: 0x57a06, // select - 0xd: 0x17b09, // draggable - 0xe: 0x3e03, // nav - 0xf: 0x17507, // command - 0x11: 0xb01, // u - 0x14: 0x2d507, // headers - 0x15: 0x44a08, // datalist - 0x17: 0x4e04, // samp - 0x1a: 0x3fb09, // onkeydown - 0x1b: 0x55f08, // onscroll - 0x1c: 0x15003, // col - 0x20: 0x3c908, // itemprop - 0x21: 0x2780a, // http-equiv - 0x22: 0x61d03, // sup - 0x24: 0x1d008, // required - 0x2b: 0x25e07, // preload - 0x2c: 0x6040d, // onbeforeprint - 0x2d: 0x3600b, // ondragenter - 0x2e: 0x50902, // dt - 0x2f: 0x5a708, // onsubmit - 0x30: 0x27002, // hr - 0x31: 0x32f0d, // oncontextmenu - 0x33: 0x29c05, // image - 0x34: 0x50d07, // onpause - 0x35: 0x25906, // hgroup - 0x36: 0x7704, // ping - 0x37: 0x57808, // onselect - 0x3a: 0x11303, // div - 0x3b: 0x1fa0e, // onautocomplete - 0x40: 0x2eb02, // mi - 0x41: 0x31c08, // seamless - 0x42: 0x2807, // charset - 0x43: 0x8502, // id - 0x44: 0x5200a, // onpopstate - 0x45: 0x3ef03, // del - 0x46: 0x2cb07, // marquee - 0x47: 0x3309, // accesskey - 0x49: 0x8d06, // footer - 0x4a: 0x44e04, // list - 0x4b: 0x2b005, // ismap - 0x51: 0x33804, // menu - 0x52: 0x2f04, // body - 0x55: 0x9a08, // frameset - 0x56: 0x54a07, // onreset - 0x57: 0x12705, // blink - 0x58: 0xa105, // title - 0x59: 0x38807, // article - 0x5b: 0x22e02, // th - 0x5d: 0x13101, // q - 0x5e: 0x3cf04, // open - 0x5f: 0x2fa04, // area - 0x61: 0x44206, // onload - 0x62: 0xda04, // font - 0x63: 0xd604, // base - 0x64: 0x16207, // colspan - 0x65: 0x53707, // keytype - 0x66: 0x11e02, // dl - 0x68: 0x1b008, // fieldset - 0x6a: 0x2eb03, // min - 0x6b: 0x11503, // var - 0x6f: 0x2d506, // header - 0x70: 0x13f02, // rt - 0x71: 0x15008, // colgroup - 0x72: 0x23502, // mn - 0x74: 0x13a07, // onabort - 0x75: 0x3906, // keygen - 0x76: 0x4c209, // onoffline - 0x77: 0x21f09, // challenge - 0x78: 0x2b203, // map - 0x7a: 0x2e902, // h4 - 0x7b: 0x3b607, // onerror - 0x7c: 0x2e109, // maxlength - 0x7d: 0x2f505, // mtext - 0x7e: 0xbb07, // sandbox - 0x7f: 0x58b06, // onsort - 0x80: 0x100a, // malignmark - 0x81: 0x45d04, // meta - 0x82: 0x7b05, // async - 0x83: 0x2a702, // h3 - 0x84: 0x26702, // dd - 0x85: 0x27004, // href - 0x86: 0x6e0a, // mediagroup - 0x87: 0x19406, // coords - 0x88: 0x41107, // srclang - 0x89: 0x34d0a, // ondblclick - 0x8a: 0x4005, // value - 0x8c: 0xe908, // oncancel - 0x8e: 0x3230a, // spellcheck - 0x8f: 0x9a05, // frame - 0x91: 0x12403, // big - 0x94: 0x1f606, // action - 0x95: 0x6903, // dir - 0x97: 0x2fb08, // readonly - 0x99: 0x42d05, // table - 0x9a: 0x61607, // summary - 0x9b: 0x12103, // wbr - 0x9c: 0x30a, // radiogroup - 0x9d: 0x6c04, // name - 0x9f: 0x62306, // system - 0xa1: 0x15d05, // color - 0xa2: 0x7f06, // canvas - 0xa3: 0x25504, // html - 0xa5: 0x56f09, // onseeking - 0xac: 0x4f905, // shape - 0xad: 0x25f03, // rel - 0xae: 0x28510, // oncanplaythrough - 0xaf: 0x3760a, // ondragover - 0xb0: 0x62608, // template - 0xb1: 0x1d80d, // foreignObject - 0xb3: 0x9204, // rows - 0xb6: 0x44e07, // listing - 0xb7: 0x49c06, // output - 0xb9: 0x3310b, // contextmenu - 0xbb: 0x11f03, // low - 0xbc: 0x1c602, // rp - 0xbd: 0x5bb09, // onsuspend - 0xbe: 0x13606, // button - 0xbf: 0x4db04, // desc - 0xc1: 0x4e207, // section - 0xc2: 0x52a0a, // onprogress - 0xc3: 0x59e09, // onstorage - 0xc4: 0x2d204, // math - 0xc5: 0x4503, // alt - 0xc7: 0x8a02, // ul - 0xc8: 0x5107, // pattern - 0xc9: 0x4b60c, // onmousewheel - 0xca: 0x35709, // ondragend - 0xcb: 0xaf04, // ruby - 0xcc: 0xc01, // p - 0xcd: 0x31707, // onclose - 0xce: 0x24205, // meter - 0xcf: 0x11807, // bgsound - 0xd2: 0x25106, // height - 0xd4: 0x101, // b - 0xd5: 0x2c308, // itemtype - 0xd8: 0x1bb07, // caption - 0xd9: 0x10c08, // disabled - 0xdb: 0x33808, // menuitem - 0xdc: 0x62003, // svg - 0xdd: 0x18f05, // small - 0xde: 0x44a04, // data - 0xe0: 0x4cb08, // ononline - 0xe1: 0x2a206, // mglyph - 0xe3: 0x6505, // embed - 0xe4: 0x10502, // tr - 0xe5: 0x46b0b, // onloadstart - 0xe7: 0x3c306, // srcdoc - 0xeb: 0x5c408, // ontoggle - 0xed: 0xe703, // bdo - 0xee: 0x4702, // td - 0xef: 0x8305, // aside - 0xf0: 0x29402, // h2 - 0xf1: 0x52c08, // progress - 0xf2: 0x12c0a, // blockquote - 0xf4: 0xf005, // label - 0xf5: 0x601, // i - 0xf7: 0x9207, // rowspan - 0xfb: 0x51709, // onplaying - 0xfd: 0x2a103, // img - 0xfe: 0xf608, // optgroup - 0xff: 0x42307, // content - 0x101: 0x53e0c, // onratechange - 0x103: 0x3da0c, // onhashchange - 0x104: 0x4807, // details - 0x106: 0x40008, // download - 0x109: 0x14009, // translate - 0x10b: 0x4230f, // contenteditable - 0x10d: 0x36b0b, // ondragleave - 0x10e: 0x2106, // accept - 0x10f: 0x57a08, // selected - 0x112: 0x1f20a, // formaction - 0x113: 0x5b506, // center - 0x115: 0x45510, // onloadedmetadata - 0x116: 0x12804, // link - 0x117: 0xdd04, // time - 0x118: 0x19f0b, // crossorigin - 0x119: 0x3bd07, // onfocus - 0x11a: 0x58704, // wrap - 0x11b: 0x42204, // icon - 0x11d: 0x28105, // video - 0x11e: 0x4de05, // class - 0x121: 0x5d40e, // onvolumechange - 0x122: 0xaa06, // onblur - 0x123: 0x2b909, // itemscope - 0x124: 0x61105, // style - 0x127: 0x41e06, // public - 0x129: 0x2320e, // formnovalidate - 0x12a: 0x58206, // onshow - 0x12c: 0x51706, // onplay - 0x12d: 0x3c804, // cite - 0x12e: 0x2bc02, // ms - 0x12f: 0xdb0c, // ontimeupdate - 0x130: 0x10904, // kind - 0x131: 0x2470a, // formtarget - 0x135: 0x3af07, // onended - 0x136: 0x26506, // hidden - 0x137: 0x2c01, // s - 0x139: 0x2280a, // formmethod - 0x13a: 0x3e805, // input - 0x13c: 0x50b02, // h6 - 0x13d: 0xc902, // ol - 0x13e: 0x3420b, // oncuechange - 0x13f: 0x1e50d, // foreignobject - 0x143: 0x4e70e, // onbeforeunload - 0x144: 0x2bd05, // scope - 0x145: 0x39609, // onemptied - 0x146: 0x14b05, // defer - 0x147: 0xc103, // xmp - 0x148: 0x39f10, // ondurationchange - 0x149: 0x1903, // kbd - 0x14c: 0x47609, // onmessage - 0x14d: 0x60006, // option - 0x14e: 0x2eb09, // minlength - 0x14f: 0x32807, // checked - 0x150: 0xce08, // autoplay - 0x152: 0x202, // br - 0x153: 0x2360a, // novalidate - 0x156: 0x6307, // noembed - 0x159: 0x31007, // onclick - 0x15a: 0x47f0b, // onmousedown - 0x15b: 0x3a708, // onchange - 0x15e: 0x3f209, // oninvalid - 0x15f: 0x2bd06, // scoped - 0x160: 0x18808, // controls - 0x161: 0x30b05, // muted - 0x162: 0x58d08, // sortable - 0x163: 0x51106, // usemap - 0x164: 0x1b80a, // figcaption - 0x165: 0x35706, // ondrag - 0x166: 0x26b04, // high - 0x168: 0x3c303, // src - 0x169: 0x15706, // poster - 0x16b: 0x1670e, // annotation-xml - 0x16c: 0x5f704, // step - 0x16d: 0x4, // abbr - 0x16e: 0x1b06, // dialog - 0x170: 0x1202, // li - 0x172: 0x3ed02, // mo - 0x175: 0x1d803, // for - 0x176: 0x1a803, // ins - 0x178: 0x55504, // size - 0x179: 0x43210, // onlanguagechange - 0x17a: 0x8607, // default - 0x17b: 0x1a03, // bdi - 0x17c: 0x4d30a, // onpagehide - 0x17d: 0x6907, // dirname - 0x17e: 0x21404, // type - 0x17f: 0x1f204, // form - 0x181: 0x28509, // oncanplay - 0x182: 0x6103, // dfn - 0x183: 0x46308, // tabindex - 0x186: 0x6502, // em - 0x187: 0x27404, // lang - 0x189: 0x39108, // dropzone - 0x18a: 0x4080a, // onkeypress - 0x18b: 0x23c08, // datetime - 0x18c: 0x16204, // cols - 0x18d: 0x1, // a - 0x18e: 0x4420c, // onloadeddata - 0x190: 0xa605, // audio - 0x192: 0x2e05, // tbody - 0x193: 0x22c06, // method - 0x195: 0xf404, // loop - 0x196: 0x29606, // iframe - 0x198: 0x2d504, // head - 0x19e: 0x5f108, // manifest - 0x19f: 0xb309, // autofocus - 0x1a0: 0x14904, // code - 0x1a1: 0x55906, // strong - 0x1a2: 0x30308, // multiple - 0x1a3: 0xc05, // param - 0x1a6: 0x21107, // enctype - 0x1a7: 0x5b304, // face - 0x1a8: 0xfd09, // plaintext - 0x1a9: 0x26e02, // h1 - 0x1aa: 0x59509, // onstalled - 0x1ad: 0x3d406, // script - 0x1ae: 0x2db06, // spacer - 0x1af: 0x55108, // onresize - 0x1b0: 0x4a20b, // onmouseover - 0x1b1: 0x5cc08, // onunload - 0x1b2: 0x56708, // onseeked - 0x1b4: 0x2140d, // typemustmatch - 0x1b5: 0x1cc06, // figure - 0x1b6: 0x4950a, // onmouseout - 0x1b7: 0x25e03, // pre - 0x1b8: 0x50705, // width - 0x1b9: 0x19906, // sorted - 0x1bb: 0x5704, // nobr - 0x1be: 0x5302, // tt - 0x1bf: 0x1105, // align - 0x1c0: 0x3e607, // oninput - 0x1c3: 0x41807, // onkeyup - 0x1c6: 0x1c00c, // onafterprint - 0x1c7: 0x210e, // accept-charset - 0x1c8: 0x33c06, // itemid - 0x1c9: 0x3e809, // inputmode - 0x1cb: 0x53306, // strike - 0x1cc: 0x5a903, // sub - 0x1cd: 0x10505, // track - 0x1ce: 0x38605, // start - 0x1d0: 0xd608, // basefont - 0x1d6: 0x1aa06, // source - 0x1d7: 0x18206, // legend - 0x1d8: 0x2d405, // thead - 0x1da: 0x8c05, // tfoot - 0x1dd: 0x1ec06, // object - 0x1de: 0x6e05, // media - 0x1df: 0x1670a, // annotation - 0x1e0: 0x20d0b, // formenctype - 0x1e2: 0x3d208, // noscript - 0x1e4: 0x55505, // sizes - 0x1e5: 0x1fc0c, // autocomplete - 0x1e6: 0x9504, // span - 0x1e7: 0x9808, // noframes - 0x1e8: 0x24b06, // target - 0x1e9: 0x38f06, // ondrop - 0x1ea: 0x2b306, // applet - 0x1ec: 0x5a08, // reversed - 0x1f0: 0x2a907, // isindex - 0x1f3: 0x27008, // hreflang - 0x1f5: 0x2f302, // h5 - 0x1f6: 0x4f307, // address - 0x1fa: 0x2e103, // max - 0x1fb: 0xc30b, // placeholder - 0x1fc: 0x2f608, // textarea - 0x1fe: 0x4ad09, // onmouseup - 0x1ff: 0x3800b, // ondragstart + 0x1: 0xe60a, // mediagroup + 0x2: 0x2e404, // lang + 0x4: 0x2c09, // accesskey + 0x5: 0x8b08, // frameset + 0x7: 0x63a08, // onselect + 0x8: 0x71106, // system + 0xa: 0x64905, // width + 0xc: 0x2890b, // formenctype + 0xd: 0x13702, // ol + 0xe: 0x3970b, // oncuechange + 0x10: 0x14b03, // bdo + 0x11: 0x11505, // audio + 0x12: 0x17a09, // draggable + 0x14: 0x2f105, // video + 0x15: 0x2b102, // mn + 0x16: 0x38704, // menu + 0x17: 0x2cf06, // poster + 0x19: 0xf606, // footer + 0x1a: 0x2a806, // method + 0x1b: 0x2b808, // datetime + 0x1c: 0x19507, // onabort + 0x1d: 0x460e, // updateviacache + 0x1e: 0xff05, // async + 0x1f: 0x49d06, // onload + 0x21: 0x11908, // oncancel + 0x22: 0x62908, // onseeked + 0x23: 0x30205, // image + 0x24: 0x5d812, // onrejectionhandled + 0x26: 0x17404, // link + 0x27: 0x51d06, // output + 0x28: 0x33104, // head + 0x29: 0x4ff0c, // onmouseleave + 0x2a: 0x57f07, // onpaste + 0x2b: 0x5a409, // onplaying + 0x2c: 0x1c407, // colspan + 0x2f: 0x1bf05, // color + 0x30: 0x5f504, // size + 0x31: 0x2e80a, // http-equiv + 0x33: 0x601, // i + 0x34: 0x5590a, // onpagehide + 0x35: 0x68c14, // onunhandledrejection + 0x37: 0x42a07, // onerror + 0x3a: 0x3b08, // basefont + 0x3f: 0x1303, // nav + 0x40: 0x17704, // kind + 0x41: 0x35708, // readonly + 0x42: 0x30806, // mglyph + 0x44: 0xb202, // li + 0x46: 0x2d506, // hidden + 0x47: 0x70e03, // svg + 0x48: 0x58304, // step + 0x49: 0x23f09, // integrity + 0x4a: 0x58606, // public + 0x4c: 0x1ab03, // col + 0x4d: 0x1870a, // blockquote + 0x4e: 0x34f02, // h5 + 0x50: 0x5b908, // progress + 0x51: 0x5f505, // sizes + 0x52: 0x34502, // h4 + 0x56: 0x33005, // thead + 0x57: 0xd607, // keytype + 0x58: 0x5b70a, // onprogress + 0x59: 0x44b09, // inputmode + 0x5a: 0x3b109, // ondragend + 0x5d: 0x3a205, // oncut + 0x5e: 0x43706, // spacer + 0x5f: 0x1ab08, // colgroup + 0x62: 0x16502, // is + 0x65: 0x3c02, // as + 0x66: 0x54809, // onoffline + 0x67: 0x33706, // sorted + 0x69: 0x48d10, // onlanguagechange + 0x6c: 0x43d0c, // onhashchange + 0x6d: 0x9604, // name + 0x6e: 0xf505, // tfoot + 0x6f: 0x56104, // desc + 0x70: 0x33d03, // max + 0x72: 0x1ea06, // coords + 0x73: 0x30d02, // h3 + 0x74: 0x6e70e, // onbeforeunload + 0x75: 0x9c04, // rows + 0x76: 0x63c06, // select + 0x77: 0x9805, // meter + 0x78: 0x38b06, // itemid + 0x79: 0x53c0c, // onmousewheel + 0x7a: 0x5c006, // srcdoc + 0x7d: 0x1ba05, // track + 0x7f: 0x31f08, // itemtype + 0x82: 0xa402, // mo + 0x83: 0x41b08, // onchange + 0x84: 0x33107, // headers + 0x85: 0x5cc0c, // onratechange + 0x86: 0x60819, // onsecuritypolicyviolation + 0x88: 0x4a508, // datalist + 0x89: 0x4e80b, // onmousedown + 0x8a: 0x1ef04, // slot + 0x8b: 0x4b010, // onloadedmetadata + 0x8c: 0x1a06, // accept + 0x8d: 0x26806, // object + 0x91: 0x6b30e, // onvolumechange + 0x92: 0x2107, // charset + 0x93: 0x27613, // onautocompleteerror + 0x94: 0xc113, // allowpaymentrequest + 0x95: 0x2804, // body + 0x96: 0x10a07, // default + 0x97: 0x63c08, // selected + 0x98: 0x21e04, // face + 0x99: 0x1e505, // shape + 0x9b: 0x68408, // ontoggle + 0x9e: 0x64b02, // dt + 0x9f: 0xb604, // mark + 0xa1: 0xb01, // u + 0xa4: 0x6ab08, // onunload + 0xa5: 0x5d04, // loop + 0xa6: 0x16408, // disabled + 0xaa: 0x42307, // onended + 0xab: 0xb00a, // malignmark + 0xad: 0x67b09, // onsuspend + 0xae: 0x35105, // mtext + 0xaf: 0x64f06, // onsort + 0xb0: 0x19d08, // itemprop + 0xb3: 0x67109, // itemscope + 0xb4: 0x17305, // blink + 0xb6: 0x3b106, // ondrag + 0xb7: 0xa702, // ul + 0xb8: 0x26e04, // form + 0xb9: 0x12907, // sandbox + 0xba: 0x8b05, // frame + 0xbb: 0x1505, // value + 0xbc: 0x66209, // onstorage + 0xbf: 0xaa07, // acronym + 0xc0: 0x19a02, // rt + 0xc2: 0x202, // br + 0xc3: 0x22608, // fieldset + 0xc4: 0x2900d, // typemustmatch + 0xc5: 0xa208, // nomodule + 0xc6: 0x6c07, // noembed + 0xc7: 0x69e0d, // onbeforeprint + 0xc8: 0x19106, // button + 0xc9: 0x2f507, // onclick + 0xca: 0x70407, // summary + 0xcd: 0xfb04, // ruby + 0xce: 0x56405, // class + 0xcf: 0x3f40b, // ondragstart + 0xd0: 0x23107, // caption + 0xd4: 0xdd0e, // allowusermedia + 0xd5: 0x4cf0b, // onloadstart + 0xd9: 0x16b03, // div + 0xda: 0x4a904, // list + 0xdb: 0x32e04, // math + 0xdc: 0x44b05, // input + 0xdf: 0x3ea0a, // ondragover + 0xe0: 0x2de02, // h2 + 0xe2: 0x1b209, // plaintext + 0xe4: 0x4f30c, // onmouseenter + 0xe7: 0x47907, // checked + 0xe8: 0x47003, // pre + 0xea: 0x35f08, // multiple + 0xeb: 0xba03, // bdi + 0xec: 0x33d09, // maxlength + 0xed: 0xcf01, // q + 0xee: 0x61f0a, // onauxclick + 0xf0: 0x57c03, // wbr + 0xf2: 0x3b04, // base + 0xf3: 0x6e306, // option + 0xf5: 0x41310, // ondurationchange + 0xf7: 0x8908, // noframes + 0xf9: 0x40508, // dropzone + 0xfb: 0x67505, // scope + 0xfc: 0x8008, // reversed + 0xfd: 0x3ba0b, // ondragenter + 0xfe: 0x3fa05, // start + 0xff: 0x12f03, // xmp + 0x100: 0x5f907, // srclang + 0x101: 0x30703, // img + 0x104: 0x101, // b + 0x105: 0x25403, // for + 0x106: 0x10705, // aside + 0x107: 0x44907, // oninput + 0x108: 0x35604, // area + 0x109: 0x2a40a, // formmethod + 0x10a: 0x72604, // wrap + 0x10c: 0x23c02, // rp + 0x10d: 0x46b0a, // onkeypress + 0x10e: 0x6802, // tt + 0x110: 0x34702, // mi + 0x111: 0x36705, // muted + 0x112: 0xf303, // alt + 0x113: 0x5c504, // code + 0x114: 0x6e02, // em + 0x115: 0x3c50a, // ondragexit + 0x117: 0x9f04, // span + 0x119: 0x6d708, // manifest + 0x11a: 0x38708, // menuitem + 0x11b: 0x58b07, // content + 0x11d: 0x6c109, // onwaiting + 0x11f: 0x4c609, // onloadend + 0x121: 0x37e0d, // oncontextmenu + 0x123: 0x56d06, // onblur + 0x124: 0x3fc07, // article + 0x125: 0x9303, // dir + 0x126: 0xef04, // ping + 0x127: 0x24c08, // required + 0x128: 0x45509, // oninvalid + 0x129: 0xb105, // align + 0x12b: 0x58a04, // icon + 0x12c: 0x64d02, // h6 + 0x12d: 0x1c404, // cols + 0x12e: 0x22e0a, // figcaption + 0x12f: 0x45e09, // onkeydown + 0x130: 0x66b08, // onsubmit + 0x131: 0x14d09, // oncanplay + 0x132: 0x70b03, // sup + 0x133: 0xc01, // p + 0x135: 0x40a09, // onemptied + 0x136: 0x39106, // oncopy + 0x137: 0x19c04, // cite + 0x138: 0x3a70a, // ondblclick + 0x13a: 0x50b0b, // onmousemove + 0x13c: 0x66d03, // sub + 0x13d: 0x48703, // rel + 0x13e: 0x5f08, // optgroup + 0x142: 0x9c07, // rowspan + 0x143: 0x37806, // source + 0x144: 0x21608, // noscript + 0x145: 0x1a304, // open + 0x146: 0x20403, // ins + 0x147: 0x2540d, // foreignObject + 0x148: 0x5ad0a, // onpopstate + 0x14a: 0x28d07, // enctype + 0x14b: 0x2760e, // onautocomplete + 0x14c: 0x35208, // textarea + 0x14e: 0x2780c, // autocomplete + 0x14f: 0x15702, // hr + 0x150: 0x1de08, // controls + 0x151: 0x10902, // id + 0x153: 0x2360c, // onafterprint + 0x155: 0x2610d, // foreignobject + 0x156: 0x32707, // marquee + 0x157: 0x59a07, // onpause + 0x158: 0x5e602, // dl + 0x159: 0x5206, // height + 0x15a: 0x34703, // min + 0x15b: 0x9307, // dirname + 0x15c: 0x1f209, // translate + 0x15d: 0x5604, // html + 0x15e: 0x34709, // minlength + 0x15f: 0x48607, // preload + 0x160: 0x71408, // template + 0x161: 0x3df0b, // ondragleave + 0x162: 0x3a02, // rb + 0x164: 0x5c003, // src + 0x165: 0x6dd06, // strong + 0x167: 0x7804, // samp + 0x168: 0x6f307, // address + 0x169: 0x55108, // ononline + 0x16b: 0x1310b, // placeholder + 0x16c: 0x2c406, // target + 0x16d: 0x20605, // small + 0x16e: 0x6ca07, // onwheel + 0x16f: 0x1c90a, // annotation + 0x170: 0x4740a, // spellcheck + 0x171: 0x7207, // details + 0x172: 0x10306, // canvas + 0x173: 0x12109, // autofocus + 0x174: 0xc05, // param + 0x176: 0x46308, // download + 0x177: 0x45203, // del + 0x178: 0x36c07, // onclose + 0x179: 0xb903, // kbd + 0x17a: 0x31906, // applet + 0x17b: 0x2e004, // href + 0x17c: 0x5f108, // onresize + 0x17e: 0x49d0c, // onloadeddata + 0x180: 0xcc02, // tr + 0x181: 0x2c00a, // formtarget + 0x182: 0x11005, // title + 0x183: 0x6ff05, // style + 0x184: 0xd206, // strike + 0x185: 0x59e06, // usemap + 0x186: 0x2fc06, // iframe + 0x187: 0x1004, // main + 0x189: 0x7b07, // picture + 0x18c: 0x31605, // ismap + 0x18e: 0x4a504, // data + 0x18f: 0x5905, // label + 0x191: 0x3d10e, // referrerpolicy + 0x192: 0x15602, // th + 0x194: 0x53606, // prompt + 0x195: 0x56807, // section + 0x197: 0x6d107, // optimum + 0x198: 0x2db04, // high + 0x199: 0x15c02, // h1 + 0x19a: 0x65909, // onstalled + 0x19b: 0x16d03, // var + 0x19c: 0x4204, // time + 0x19e: 0x67402, // ms + 0x19f: 0x33106, // header + 0x1a0: 0x4da09, // onmessage + 0x1a1: 0x1a605, // nonce + 0x1a2: 0x26e0a, // formaction + 0x1a3: 0x22006, // center + 0x1a4: 0x3704, // nobr + 0x1a5: 0x59505, // table + 0x1a6: 0x4a907, // listing + 0x1a7: 0x18106, // legend + 0x1a9: 0x29b09, // challenge + 0x1aa: 0x24806, // figure + 0x1ab: 0xe605, // media + 0x1ae: 0xd904, // type + 0x1af: 0x3f04, // font + 0x1b0: 0x4da0e, // onmessageerror + 0x1b1: 0x37108, // seamless + 0x1b2: 0x8703, // dfn + 0x1b3: 0x5c705, // defer + 0x1b4: 0xc303, // low + 0x1b5: 0x19a03, // rtc + 0x1b6: 0x5230b, // onmouseover + 0x1b7: 0x2b20a, // novalidate + 0x1b8: 0x71c0a, // workertype + 0x1ba: 0x3cd07, // itemref + 0x1bd: 0x1, // a + 0x1be: 0x31803, // map + 0x1bf: 0x400c, // ontimeupdate + 0x1c0: 0x15e07, // bgsound + 0x1c1: 0x3206, // keygen + 0x1c2: 0x2705, // tbody + 0x1c5: 0x64406, // onshow + 0x1c7: 0x2501, // s + 0x1c8: 0x6607, // pattern + 0x1cc: 0x14d10, // oncanplaythrough + 0x1ce: 0x2d702, // dd + 0x1cf: 0x6f906, // srcset + 0x1d0: 0x17003, // big + 0x1d2: 0x65108, // sortable + 0x1d3: 0x48007, // onkeyup + 0x1d5: 0x5a406, // onplay + 0x1d7: 0x4b804, // meta + 0x1d8: 0x40306, // ondrop + 0x1da: 0x60008, // onscroll + 0x1db: 0x1fb0b, // crossorigin + 0x1dc: 0x5730a, // onpageshow + 0x1dd: 0x4, // abbr + 0x1de: 0x9202, // td + 0x1df: 0x58b0f, // contenteditable + 0x1e0: 0x27206, // action + 0x1e1: 0x1400b, // playsinline + 0x1e2: 0x43107, // onfocus + 0x1e3: 0x2e008, // hreflang + 0x1e5: 0x5160a, // onmouseout + 0x1e6: 0x5ea07, // onreset + 0x1e7: 0x13c08, // autoplay + 0x1e8: 0x63109, // onseeking + 0x1ea: 0x67506, // scoped + 0x1ec: 0x30a, // radiogroup + 0x1ee: 0x3800b, // contextmenu + 0x1ef: 0x52e09, // onmouseup + 0x1f1: 0x2ca06, // hgroup + 0x1f2: 0x2080f, // allowfullscreen + 0x1f3: 0x4be08, // tabindex + 0x1f6: 0x30f07, // isindex + 0x1f7: 0x1a0e, // accept-charset + 0x1f8: 0x2ae0e, // formnovalidate + 0x1fb: 0x1c90e, // annotation-xml + 0x1fc: 0x6e05, // embed + 0x1fd: 0x21806, // script + 0x1fe: 0xbb06, // dialog + 0x1ff: 0x1d707, // command } -const atomText = "abbradiogrouparamalignmarkbdialogaccept-charsetbodyaccesskey" + - "genavaluealtdetailsampatternobreversedfnoembedirnamediagroup" + - "ingasyncanvasidefaultfooterowspanoframesetitleaudionblurubya" + - "utofocusandboxmplaceholderautoplaybasefontimeupdatebdoncance" + - "labelooptgrouplaintextrackindisabledivarbgsoundlowbrbigblink" + - "blockquotebuttonabortranslatecodefercolgroupostercolorcolspa" + - "nnotation-xmlcommandraggablegendcontrolsmallcoordsortedcross" + - "originsourcefieldsetfigcaptionafterprintfigurequiredforeignO" + - "bjectforeignobjectformactionautocompleteerrorformenctypemust" + - "matchallengeformmethodformnovalidatetimeterformtargetheightm" + - "lhgroupreloadhiddenhigh1hreflanghttp-equivideoncanplaythroug" + - "h2iframeimageimglyph3isindexismappletitemscopeditemtypemarqu" + - "eematheaderspacermaxlength4minlength5mtextareadonlymultiplem" + - "utedonclickoncloseamlesspellcheckedoncontextmenuitemidoncuec" + - "hangeondblclickondragendondragenterondragleaveondragoverondr" + - "agstarticleondropzonemptiedondurationchangeonendedonerroronf" + - "ocusrcdocitempropenoscriptonhashchangeoninputmodeloninvalido" + - "nkeydownloadonkeypressrclangonkeyupublicontenteditableonlang" + - "uagechangeonloadeddatalistingonloadedmetadatabindexonloadsta" + - "rtonmessageonmousedownonmousemoveonmouseoutputonmouseoveronm" + - "ouseuponmousewheelonofflineononlineonpagehidesclassectionbef" + - "oreunloaddresshapeonpageshowidth6onpausemaponplayingonpopsta" + - "teonprogresstrikeytypeonratechangeonresetonresizestrongonscr" + - "ollonseekedonseekingonselectedonshowraponsortableonstalledon" + - "storageonsubmitemrefacenteronsuspendontoggleonunloadonvolume" + - "changeonwaitingoptimumanifestepromptoptionbeforeprintstylesu" + - "mmarysupsvgsystemplate" +const atomText = "abbradiogrouparamainavalueaccept-charsetbodyaccesskeygenobrb" + + "asefontimeupdateviacacheightmlabelooptgroupatternoembedetail" + + "sampictureversedfnoframesetdirnameterowspanomoduleacronymali" + + "gnmarkbdialogallowpaymentrequestrikeytypeallowusermediagroup" + + "ingaltfooterubyasyncanvasidefaultitleaudioncancelautofocusan" + + "dboxmplaceholderautoplaysinlinebdoncanplaythrough1bgsoundisa" + + "bledivarbigblinkindraggablegendblockquotebuttonabortcitempro" + + "penoncecolgrouplaintextrackcolorcolspannotation-xmlcommandco" + + "ntrolshapecoordslotranslatecrossoriginsmallowfullscreenoscri" + + "ptfacenterfieldsetfigcaptionafterprintegrityfigurequiredfore" + + "ignObjectforeignobjectformactionautocompleteerrorformenctype" + + "mustmatchallengeformmethodformnovalidatetimeformtargethgroup" + + "osterhiddenhigh2hreflanghttp-equivideonclickiframeimageimgly" + + "ph3isindexismappletitemtypemarqueematheadersortedmaxlength4m" + + "inlength5mtextareadonlymultiplemutedoncloseamlessourceoncont" + + "extmenuitemidoncopyoncuechangeoncutondblclickondragendondrag" + + "enterondragexitemreferrerpolicyondragleaveondragoverondragst" + + "articleondropzonemptiedondurationchangeonendedonerroronfocus" + + "paceronhashchangeoninputmodeloninvalidonkeydownloadonkeypres" + + "spellcheckedonkeyupreloadonlanguagechangeonloadeddatalisting" + + "onloadedmetadatabindexonloadendonloadstartonmessageerroronmo" + + "usedownonmouseenteronmouseleaveonmousemoveonmouseoutputonmou" + + "seoveronmouseupromptonmousewheelonofflineononlineonpagehides" + + "classectionbluronpageshowbronpastepublicontenteditableonpaus" + + "emaponplayingonpopstateonprogressrcdocodeferonratechangeonre" + + "jectionhandledonresetonresizesrclangonscrollonsecuritypolicy" + + "violationauxclickonseekedonseekingonselectedonshowidth6onsor" + + "tableonstalledonstorageonsubmitemscopedonsuspendontoggleonun" + + "handledrejectionbeforeprintonunloadonvolumechangeonwaitingon" + + "wheeloptimumanifestrongoptionbeforeunloaddressrcsetstylesumm" + + "arysupsvgsystemplateworkertypewrap" diff --git a/vendor/golang.org/x/net/html/const.go b/vendor/golang.org/x/net/html/const.go index 52f651ff..5eb7c5a8 100644 --- a/vendor/golang.org/x/net/html/const.go +++ b/vendor/golang.org/x/net/html/const.go @@ -4,7 +4,7 @@ package html -// Section 12.2.3.2 of the HTML5 specification says "The following elements +// Section 12.2.4.2 of the HTML5 specification says "The following elements // have varying levels of special parsing rules". // https://html.spec.whatwg.org/multipage/syntax.html#the-stack-of-open-elements var isSpecialElementMap = map[string]bool{ @@ -52,10 +52,12 @@ var isSpecialElementMap = map[string]bool{ "iframe": true, "img": true, "input": true, - "isindex": true, + "isindex": true, // The 'isindex' element has been removed, but keep it for backwards compatibility. + "keygen": true, "li": true, "link": true, "listing": true, + "main": true, "marquee": true, "menu": true, "meta": true, diff --git a/vendor/golang.org/x/net/html/doc.go b/vendor/golang.org/x/net/html/doc.go index 94f49687..822ed42a 100644 --- a/vendor/golang.org/x/net/html/doc.go +++ b/vendor/golang.org/x/net/html/doc.go @@ -49,18 +49,18 @@ call to Next. For example, to extract an HTML page's anchor text: for { tt := z.Next() switch tt { - case ErrorToken: + case html.ErrorToken: return z.Err() - case TextToken: + case html.TextToken: if depth > 0 { // emitBytes should copy the []byte it receives, // if it doesn't process it immediately. emitBytes(z.Text()) } - case StartTagToken, EndTagToken: + case html.StartTagToken, html.EndTagToken: tn, _ := z.TagName() if len(tn) == 1 && tn[0] == 'a' { - if tt == StartTagToken { + if tt == html.StartTagToken { depth++ } else { depth-- diff --git a/vendor/golang.org/x/net/html/foreign.go b/vendor/golang.org/x/net/html/foreign.go index d3b38440..01477a96 100644 --- a/vendor/golang.org/x/net/html/foreign.go +++ b/vendor/golang.org/x/net/html/foreign.go @@ -67,7 +67,7 @@ func mathMLTextIntegrationPoint(n *Node) bool { return false } -// Section 12.2.5.5. +// Section 12.2.6.5. var breakout = map[string]bool{ "b": true, "big": true, @@ -115,7 +115,7 @@ var breakout = map[string]bool{ "var": true, } -// Section 12.2.5.5. +// Section 12.2.6.5. var svgTagNameAdjustments = map[string]string{ "altglyph": "altGlyph", "altglyphdef": "altGlyphDef", @@ -155,7 +155,7 @@ var svgTagNameAdjustments = map[string]string{ "textpath": "textPath", } -// Section 12.2.5.1 +// Section 12.2.6.1 var mathMLAttributeAdjustments = map[string]string{ "definitionurl": "definitionURL", } diff --git a/vendor/golang.org/x/net/html/node.go b/vendor/golang.org/x/net/html/node.go index 26b657ae..2c1cade6 100644 --- a/vendor/golang.org/x/net/html/node.go +++ b/vendor/golang.org/x/net/html/node.go @@ -21,9 +21,10 @@ const ( scopeMarkerNode ) -// Section 12.2.3.3 says "scope markers are inserted when entering applet -// elements, buttons, object elements, marquees, table cells, and table -// captions, and are used to prevent formatting from 'leaking'". +// Section 12.2.4.3 says "The markers are inserted when entering applet, +// object, marquee, template, td, th, and caption elements, and are used +// to prevent formatting from "leaking" into applet, object, marquee, +// template, td, th, and caption elements". var scopeMarker = Node{Type: scopeMarkerNode} // A Node consists of a NodeType and some Data (tag name for element nodes, @@ -173,6 +174,16 @@ func (s *nodeStack) index(n *Node) int { return -1 } +// contains returns whether a is within s. +func (s *nodeStack) contains(a atom.Atom) bool { + for _, n := range *s { + if n.DataAtom == a { + return true + } + } + return false +} + // insert inserts a node at the given index. func (s *nodeStack) insert(i int, n *Node) { (*s) = append(*s, nil) @@ -191,3 +202,19 @@ func (s *nodeStack) remove(n *Node) { (*s)[j] = nil *s = (*s)[:j] } + +type insertionModeStack []insertionMode + +func (s *insertionModeStack) pop() (im insertionMode) { + i := len(*s) + im = (*s)[i-1] + *s = (*s)[:i-1] + return im +} + +func (s *insertionModeStack) top() insertionMode { + if i := len(*s); i > 0 { + return (*s)[i-1] + } + return nil +} diff --git a/vendor/golang.org/x/net/html/parse.go b/vendor/golang.org/x/net/html/parse.go index be4b2bf5..d23e05e0 100644 --- a/vendor/golang.org/x/net/html/parse.go +++ b/vendor/golang.org/x/net/html/parse.go @@ -25,20 +25,22 @@ type parser struct { hasSelfClosingToken bool // doc is the document root element. doc *Node - // The stack of open elements (section 12.2.3.2) and active formatting - // elements (section 12.2.3.3). + // The stack of open elements (section 12.2.4.2) and active formatting + // elements (section 12.2.4.3). oe, afe nodeStack - // Element pointers (section 12.2.3.4). + // Element pointers (section 12.2.4.4). head, form *Node - // Other parsing state flags (section 12.2.3.5). + // Other parsing state flags (section 12.2.4.5). scripting, framesetOK bool + // The stack of template insertion modes + templateStack insertionModeStack // im is the current insertion mode. im insertionMode // originalIM is the insertion mode to go back to after completing a text // or inTableText insertion mode. originalIM insertionMode // fosterParenting is whether new elements should be inserted according to - // the foster parenting rules (section 12.2.5.3). + // the foster parenting rules (section 12.2.6.1). fosterParenting bool // quirks is whether the parser is operating in "quirks mode." quirks bool @@ -56,7 +58,7 @@ func (p *parser) top() *Node { return p.doc } -// Stop tags for use in popUntil. These come from section 12.2.3.2. +// Stop tags for use in popUntil. These come from section 12.2.4.2. var ( defaultScopeStopTags = map[string][]a.Atom{ "": {a.Applet, a.Caption, a.Html, a.Table, a.Td, a.Th, a.Marquee, a.Object, a.Template}, @@ -79,7 +81,7 @@ const ( // popUntil pops the stack of open elements at the highest element whose tag // is in matchTags, provided there is no higher element in the scope's stop -// tags (as defined in section 12.2.3.2). It returns whether or not there was +// tags (as defined in section 12.2.4.2). It returns whether or not there was // such an element. If there was not, popUntil leaves the stack unchanged. // // For example, the set of stop tags for table scope is: "html", "table". If @@ -126,7 +128,7 @@ func (p *parser) indexOfElementInScope(s scope, matchTags ...a.Atom) int { return -1 } case tableScope: - if tagAtom == a.Html || tagAtom == a.Table { + if tagAtom == a.Html || tagAtom == a.Table || tagAtom == a.Template { return -1 } case selectScope: @@ -162,17 +164,17 @@ func (p *parser) clearStackToContext(s scope) { tagAtom := p.oe[i].DataAtom switch s { case tableScope: - if tagAtom == a.Html || tagAtom == a.Table { + if tagAtom == a.Html || tagAtom == a.Table || tagAtom == a.Template { p.oe = p.oe[:i+1] return } case tableRowScope: - if tagAtom == a.Html || tagAtom == a.Tr { + if tagAtom == a.Html || tagAtom == a.Tr || tagAtom == a.Template { p.oe = p.oe[:i+1] return } case tableBodyScope: - if tagAtom == a.Html || tagAtom == a.Tbody || tagAtom == a.Tfoot || tagAtom == a.Thead { + if tagAtom == a.Html || tagAtom == a.Tbody || tagAtom == a.Tfoot || tagAtom == a.Thead || tagAtom == a.Template { p.oe = p.oe[:i+1] return } @@ -183,7 +185,7 @@ func (p *parser) clearStackToContext(s scope) { } // generateImpliedEndTags pops nodes off the stack of open elements as long as -// the top node has a tag name of dd, dt, li, option, optgroup, p, rp, or rt. +// the top node has a tag name of dd, dt, li, optgroup, option, p, rb, rp, rt or rtc. // If exceptions are specified, nodes with that name will not be popped off. func (p *parser) generateImpliedEndTags(exceptions ...string) { var i int @@ -192,7 +194,7 @@ loop: n := p.oe[i] if n.Type == ElementNode { switch n.DataAtom { - case a.Dd, a.Dt, a.Li, a.Option, a.Optgroup, a.P, a.Rp, a.Rt: + case a.Dd, a.Dt, a.Li, a.Optgroup, a.Option, a.P, a.Rb, a.Rp, a.Rt, a.Rtc: for _, except := range exceptions { if n.Data == except { break loop @@ -207,6 +209,27 @@ loop: p.oe = p.oe[:i+1] } +// generateAllImpliedEndTags pops nodes off the stack of open elements as long as +// the top node has a tag name of caption, colgroup, dd, div, dt, li, optgroup, option, p, rb, +// rp, rt, rtc, span, tbody, td, tfoot, th, thead or tr. +func (p *parser) generateAllImpliedEndTags() { + var i int + for i = len(p.oe) - 1; i >= 0; i-- { + n := p.oe[i] + if n.Type == ElementNode { + switch n.DataAtom { + // TODO: remove this divergence from the HTML5 spec + case a.Caption, a.Colgroup, a.Dd, a.Div, a.Dt, a.Li, a.Optgroup, a.Option, a.P, a.Rb, + a.Rp, a.Rt, a.Rtc, a.Span, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr: + continue + } + } + break + } + + p.oe = p.oe[:i+1] +} + // addChild adds a child node n to the top element, and pushes n onto the stack // of open elements if it is an element node. func (p *parser) addChild(n *Node) { @@ -234,9 +257,9 @@ func (p *parser) shouldFosterParent() bool { } // fosterParent adds a child node according to the foster parenting rules. -// Section 12.2.5.3, "foster parenting". +// Section 12.2.6.1, "foster parenting". func (p *parser) fosterParent(n *Node) { - var table, parent, prev *Node + var table, parent, prev, template *Node var i int for i = len(p.oe) - 1; i >= 0; i-- { if p.oe[i].DataAtom == a.Table { @@ -245,6 +268,19 @@ func (p *parser) fosterParent(n *Node) { } } + var j int + for j = len(p.oe) - 1; j >= 0; j-- { + if p.oe[j].DataAtom == a.Template { + template = p.oe[j] + break + } + } + + if template != nil && (table == nil || j < i) { + template.AppendChild(n) + return + } + if table == nil { // The foster parent is the html element. parent = p.oe[0] @@ -304,7 +340,7 @@ func (p *parser) addElement() { }) } -// Section 12.2.3.3. +// Section 12.2.4.3. func (p *parser) addFormattingElement() { tagAtom, attr := p.tok.DataAtom, p.tok.Attr p.addElement() @@ -351,7 +387,7 @@ findIdenticalElements: p.afe = append(p.afe, p.top()) } -// Section 12.2.3.3. +// Section 12.2.4.3. func (p *parser) clearActiveFormattingElements() { for { n := p.afe.pop() @@ -361,7 +397,7 @@ func (p *parser) clearActiveFormattingElements() { } } -// Section 12.2.3.3. +// Section 12.2.4.3. func (p *parser) reconstructActiveFormattingElements() { n := p.afe.top() if n == nil { @@ -390,12 +426,12 @@ func (p *parser) reconstructActiveFormattingElements() { } } -// Section 12.2.4. +// Section 12.2.5. func (p *parser) acknowledgeSelfClosingTag() { p.hasSelfClosingToken = false } -// An insertion mode (section 12.2.3.1) is the state transition function from +// An insertion mode (section 12.2.4.1) is the state transition function from // a particular state in the HTML5 parser's state machine. It updates the // parser's fields depending on parser.tok (where ErrorToken means EOF). // It returns whether the token was consumed. @@ -403,7 +439,7 @@ type insertionMode func(*parser) bool // setOriginalIM sets the insertion mode to return to after completing a text or // inTableText insertion mode. -// Section 12.2.3.1, "using the rules for". +// Section 12.2.4.1, "using the rules for". func (p *parser) setOriginalIM() { if p.originalIM != nil { panic("html: bad parser state: originalIM was set twice") @@ -411,18 +447,38 @@ func (p *parser) setOriginalIM() { p.originalIM = p.im } -// Section 12.2.3.1, "reset the insertion mode". +// Section 12.2.4.1, "reset the insertion mode". func (p *parser) resetInsertionMode() { for i := len(p.oe) - 1; i >= 0; i-- { n := p.oe[i] - if i == 0 && p.context != nil { + last := i == 0 + if last && p.context != nil { n = p.context } switch n.DataAtom { case a.Select: + if !last { + for ancestor, first := n, p.oe[0]; ancestor != first; { + if ancestor == first { + break + } + ancestor = p.oe[p.oe.index(ancestor)-1] + switch ancestor.DataAtom { + case a.Template: + p.im = inSelectIM + return + case a.Table: + p.im = inSelectInTableIM + return + } + } + } p.im = inSelectIM case a.Td, a.Th: + // TODO: remove this divergence from the HTML5 spec. + // + // See https://bugs.chromium.org/p/chromium/issues/detail?id=829668 p.im = inCellIM case a.Tr: p.im = inRowIM @@ -434,25 +490,37 @@ func (p *parser) resetInsertionMode() { p.im = inColumnGroupIM case a.Table: p.im = inTableIM + case a.Template: + p.im = p.templateStack.top() case a.Head: - p.im = inBodyIM + // TODO: remove this divergence from the HTML5 spec. + // + // See https://bugs.chromium.org/p/chromium/issues/detail?id=829668 + p.im = inHeadIM case a.Body: p.im = inBodyIM case a.Frameset: p.im = inFramesetIM case a.Html: - p.im = beforeHeadIM + if p.head == nil { + p.im = beforeHeadIM + } else { + p.im = afterHeadIM + } default: + if last { + p.im = inBodyIM + return + } continue } return } - p.im = inBodyIM } const whitespace = " \t\r\n\f" -// Section 12.2.5.4.1. +// Section 12.2.6.4.1. func initialIM(p *parser) bool { switch p.tok.Type { case TextToken: @@ -479,7 +547,7 @@ func initialIM(p *parser) bool { return false } -// Section 12.2.5.4.2. +// Section 12.2.6.4.2. func beforeHTMLIM(p *parser) bool { switch p.tok.Type { case DoctypeToken: @@ -517,7 +585,7 @@ func beforeHTMLIM(p *parser) bool { return false } -// Section 12.2.5.4.3. +// Section 12.2.6.4.3. func beforeHeadIM(p *parser) bool { switch p.tok.Type { case TextToken: @@ -560,7 +628,7 @@ func beforeHeadIM(p *parser) bool { return false } -// Section 12.2.5.4.4. +// Section 12.2.6.4.4. func inHeadIM(p *parser) bool { switch p.tok.Type { case TextToken: @@ -590,19 +658,36 @@ func inHeadIM(p *parser) bool { case a.Head: // Ignore the token. return true + case a.Template: + p.addElement() + p.afe = append(p.afe, &scopeMarker) + p.framesetOK = false + p.im = inTemplateIM + p.templateStack = append(p.templateStack, inTemplateIM) + return true } case EndTagToken: switch p.tok.DataAtom { case a.Head: - n := p.oe.pop() - if n.DataAtom != a.Head { - panic("html: bad parser state: element not found, in the in-head insertion mode") - } + p.oe.pop() p.im = afterHeadIM return true case a.Body, a.Html, a.Br: p.parseImpliedToken(EndTagToken, a.Head, a.Head.String()) return false + case a.Template: + if !p.oe.contains(a.Template) { + return true + } + p.generateAllImpliedEndTags() + if n := p.oe.top(); n.DataAtom != a.Template { + return true + } + p.popUntil(defaultScope, a.Template) + p.clearActiveFormattingElements() + p.templateStack.pop() + p.resetInsertionMode() + return true default: // Ignore the token. return true @@ -622,7 +707,7 @@ func inHeadIM(p *parser) bool { return false } -// Section 12.2.5.4.6. +// Section 12.2.6.4.6. func afterHeadIM(p *parser) bool { switch p.tok.Type { case TextToken: @@ -648,7 +733,7 @@ func afterHeadIM(p *parser) bool { p.addElement() p.im = inFramesetIM return true - case a.Base, a.Basefont, a.Bgsound, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Title: + case a.Base, a.Basefont, a.Bgsound, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Template, a.Title: p.oe = append(p.oe, p.head) defer p.oe.remove(p.head) return inHeadIM(p) @@ -660,6 +745,8 @@ func afterHeadIM(p *parser) bool { switch p.tok.DataAtom { case a.Body, a.Html, a.Br: // Drop down to creating an implied tag. + case a.Template: + return inHeadIM(p) default: // Ignore the token. return true @@ -697,7 +784,7 @@ func copyAttributes(dst *Node, src Token) { } } -// Section 12.2.5.4.7. +// Section 12.2.6.4.7. func inBodyIM(p *parser) bool { switch p.tok.Type { case TextToken: @@ -727,10 +814,16 @@ func inBodyIM(p *parser) bool { case StartTagToken: switch p.tok.DataAtom { case a.Html: + if p.oe.contains(a.Template) { + return true + } copyAttributes(p.oe[0], p.tok) - case a.Base, a.Basefont, a.Bgsound, a.Command, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Title: + case a.Base, a.Basefont, a.Bgsound, a.Command, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Template, a.Title: return inHeadIM(p) case a.Body: + if p.oe.contains(a.Template) { + return true + } if len(p.oe) >= 2 { body := p.oe[1] if body.Type == ElementNode && body.DataAtom == a.Body { @@ -767,7 +860,7 @@ func inBodyIM(p *parser) bool { // The newline, if any, will be dealt with by the TextToken case. p.framesetOK = false case a.Form: - if p.form == nil { + if p.oe.contains(a.Template) || p.form == nil { p.popUntil(buttonScope, a.P) p.addElement() p.form = p.top() @@ -952,11 +1045,16 @@ func inBodyIM(p *parser) bool { } p.reconstructActiveFormattingElements() p.addElement() - case a.Rp, a.Rt: + case a.Rb, a.Rtc: if p.elementInScope(defaultScope, a.Ruby) { p.generateImpliedEndTags() } p.addElement() + case a.Rp, a.Rt: + if p.elementInScope(defaultScope, a.Ruby) { + p.generateImpliedEndTags("rtc") + } + p.addElement() case a.Math, a.Svg: p.reconstructActiveFormattingElements() if p.tok.DataAtom == a.Math { @@ -972,7 +1070,13 @@ func inBodyIM(p *parser) bool { p.acknowledgeSelfClosingTag() } return true - case a.Caption, a.Col, a.Colgroup, a.Frame, a.Head, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr: + case a.Frame: + // TODO: remove this divergence from the HTML5 spec. + if p.oe.contains(a.Template) { + p.addElement() + return true + } + case a.Caption, a.Col, a.Colgroup, a.Head, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr: // Ignore the token. default: p.reconstructActiveFormattingElements() @@ -993,15 +1097,28 @@ func inBodyIM(p *parser) bool { case a.Address, a.Article, a.Aside, a.Blockquote, a.Button, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Listing, a.Menu, a.Nav, a.Ol, a.Pre, a.Section, a.Summary, a.Ul: p.popUntil(defaultScope, p.tok.DataAtom) case a.Form: - node := p.form - p.form = nil - i := p.indexOfElementInScope(defaultScope, a.Form) - if node == nil || i == -1 || p.oe[i] != node { - // Ignore the token. - return true + if p.oe.contains(a.Template) { + if !p.oe.contains(a.Form) { + // Ignore the token. + return true + } + p.generateImpliedEndTags() + if p.tok.DataAtom == a.Form { + // Ignore the token. + return true + } + p.popUntil(defaultScope, a.Form) + } else { + node := p.form + p.form = nil + i := p.indexOfElementInScope(defaultScope, a.Form) + if node == nil || i == -1 || p.oe[i] != node { + // Ignore the token. + return true + } + p.generateImpliedEndTags() + p.oe.remove(node) } - p.generateImpliedEndTags() - p.oe.remove(node) case a.P: if !p.elementInScope(buttonScope, a.P) { p.parseImpliedToken(StartTagToken, a.P, a.P.String()) @@ -1022,6 +1139,8 @@ func inBodyIM(p *parser) bool { case a.Br: p.tok.Type = StartTagToken return false + case a.Template: + return inHeadIM(p) default: p.inBodyEndTagOther(p.tok.DataAtom) } @@ -1030,6 +1149,21 @@ func inBodyIM(p *parser) bool { Type: CommentNode, Data: p.tok.Data, }) + case ErrorToken: + // TODO: remove this divergence from the HTML5 spec. + if len(p.templateStack) > 0 { + p.im = inTemplateIM + return false + } else { + for _, e := range p.oe { + switch e.DataAtom { + case a.Dd, a.Dt, a.Li, a.Optgroup, a.Option, a.P, a.Rb, a.Rp, a.Rt, a.Rtc, a.Tbody, a.Td, a.Tfoot, a.Th, + a.Thead, a.Tr, a.Body, a.Html: + default: + return true + } + } + } } return true @@ -1135,6 +1269,12 @@ func (p *parser) inBodyEndTagFormatting(tagAtom a.Atom) { switch commonAncestor.DataAtom { case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr: p.fosterParent(lastNode) + case a.Template: + // TODO: remove namespace checking + if commonAncestor.Namespace == "html" { + commonAncestor = commonAncestor.LastChild + } + fallthrough default: commonAncestor.AppendChild(lastNode) } @@ -1160,7 +1300,7 @@ func (p *parser) inBodyEndTagFormatting(tagAtom a.Atom) { } // inBodyEndTagOther performs the "any other end tag" algorithm for inBodyIM. -// "Any other end tag" handling from 12.2.5.5 The rules for parsing tokens in foreign content +// "Any other end tag" handling from 12.2.6.5 The rules for parsing tokens in foreign content // https://html.spec.whatwg.org/multipage/syntax.html#parsing-main-inforeign func (p *parser) inBodyEndTagOther(tagAtom a.Atom) { for i := len(p.oe) - 1; i >= 0; i-- { @@ -1174,7 +1314,7 @@ func (p *parser) inBodyEndTagOther(tagAtom a.Atom) { } } -// Section 12.2.5.4.8. +// Section 12.2.6.4.8. func textIM(p *parser) bool { switch p.tok.Type { case ErrorToken: @@ -1203,7 +1343,7 @@ func textIM(p *parser) bool { return p.tok.Type == EndTagToken } -// Section 12.2.5.4.9. +// Section 12.2.6.4.9. func inTableIM(p *parser) bool { switch p.tok.Type { case ErrorToken: @@ -1249,7 +1389,7 @@ func inTableIM(p *parser) bool { } // Ignore the token. return true - case a.Style, a.Script: + case a.Style, a.Script, a.Template: return inHeadIM(p) case a.Input: for _, t := range p.tok.Attr { @@ -1261,7 +1401,7 @@ func inTableIM(p *parser) bool { } // Otherwise drop down to the default action. case a.Form: - if p.form != nil { + if p.oe.contains(a.Template) || p.form != nil { // Ignore the token. return true } @@ -1291,6 +1431,8 @@ func inTableIM(p *parser) bool { case a.Body, a.Caption, a.Col, a.Colgroup, a.Html, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr: // Ignore the token. return true + case a.Template: + return inHeadIM(p) } case CommentToken: p.addChild(&Node{ @@ -1309,7 +1451,7 @@ func inTableIM(p *parser) bool { return inBodyIM(p) } -// Section 12.2.5.4.11. +// Section 12.2.6.4.11. func inCaptionIM(p *parser) bool { switch p.tok.Type { case StartTagToken: @@ -1355,7 +1497,7 @@ func inCaptionIM(p *parser) bool { return inBodyIM(p) } -// Section 12.2.5.4.12. +// Section 12.2.6.4.12. func inColumnGroupIM(p *parser) bool { switch p.tok.Type { case TextToken: @@ -1386,11 +1528,13 @@ func inColumnGroupIM(p *parser) bool { p.oe.pop() p.acknowledgeSelfClosingTag() return true + case a.Template: + return inHeadIM(p) } case EndTagToken: switch p.tok.DataAtom { case a.Colgroup: - if p.oe.top().DataAtom != a.Html { + if p.oe.top().DataAtom == a.Colgroup { p.oe.pop() p.im = inTableIM } @@ -1398,17 +1542,19 @@ func inColumnGroupIM(p *parser) bool { case a.Col: // Ignore the token. return true + case a.Template: + return inHeadIM(p) } } - if p.oe.top().DataAtom != a.Html { - p.oe.pop() - p.im = inTableIM - return false + if p.oe.top().DataAtom != a.Colgroup { + return true } - return true + p.oe.pop() + p.im = inTableIM + return false } -// Section 12.2.5.4.13. +// Section 12.2.6.4.13. func inTableBodyIM(p *parser) bool { switch p.tok.Type { case StartTagToken: @@ -1460,7 +1606,7 @@ func inTableBodyIM(p *parser) bool { return inTableIM(p) } -// Section 12.2.5.4.14. +// Section 12.2.6.4.14. func inRowIM(p *parser) bool { switch p.tok.Type { case StartTagToken: @@ -1511,7 +1657,7 @@ func inRowIM(p *parser) bool { return inTableIM(p) } -// Section 12.2.5.4.15. +// Section 12.2.6.4.15. func inCellIM(p *parser) bool { switch p.tok.Type { case StartTagToken: @@ -1560,7 +1706,7 @@ func inCellIM(p *parser) bool { return inBodyIM(p) } -// Section 12.2.5.4.16. +// Section 12.2.6.4.16. func inSelectIM(p *parser) bool { switch p.tok.Type { case ErrorToken: @@ -1597,7 +1743,7 @@ func inSelectIM(p *parser) bool { p.tokenizer.NextIsNotRawText() // Ignore the token. return true - case a.Script: + case a.Script, a.Template: return inHeadIM(p) } case EndTagToken: @@ -1618,6 +1764,8 @@ func inSelectIM(p *parser) bool { if p.popUntil(selectScope, a.Select) { p.resetInsertionMode() } + case a.Template: + return inHeadIM(p) } case CommentToken: p.addChild(&Node{ @@ -1632,7 +1780,7 @@ func inSelectIM(p *parser) bool { return true } -// Section 12.2.5.4.17. +// Section 12.2.6.4.17. func inSelectInTableIM(p *parser) bool { switch p.tok.Type { case StartTagToken, EndTagToken: @@ -1650,7 +1798,62 @@ func inSelectInTableIM(p *parser) bool { return inSelectIM(p) } -// Section 12.2.5.4.18. +// Section 12.2.6.4.18. +func inTemplateIM(p *parser) bool { + switch p.tok.Type { + case TextToken, CommentToken, DoctypeToken: + return inBodyIM(p) + case StartTagToken: + switch p.tok.DataAtom { + case a.Base, a.Basefont, a.Bgsound, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Template, a.Title: + return inHeadIM(p) + case a.Caption, a.Colgroup, a.Tbody, a.Tfoot, a.Thead: + p.templateStack.pop() + p.templateStack = append(p.templateStack, inTableIM) + p.im = inTableIM + return false + case a.Col: + p.templateStack.pop() + p.templateStack = append(p.templateStack, inColumnGroupIM) + p.im = inColumnGroupIM + return false + case a.Tr: + p.templateStack.pop() + p.templateStack = append(p.templateStack, inTableBodyIM) + p.im = inTableBodyIM + return false + case a.Td, a.Th: + p.templateStack.pop() + p.templateStack = append(p.templateStack, inRowIM) + p.im = inRowIM + return false + default: + p.templateStack.pop() + p.templateStack = append(p.templateStack, inBodyIM) + p.im = inBodyIM + return false + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Template: + return inHeadIM(p) + default: + // Ignore the token. + return true + } + } + if !p.oe.contains(a.Template) { + // Ignore the token. + return true + } + p.popUntil(defaultScope, a.Template) + p.clearActiveFormattingElements() + p.templateStack.pop() + p.resetInsertionMode() + return false +} + +// Section 12.2.6.4.19. func afterBodyIM(p *parser) bool { switch p.tok.Type { case ErrorToken: @@ -1688,7 +1891,7 @@ func afterBodyIM(p *parser) bool { return false } -// Section 12.2.5.4.19. +// Section 12.2.6.4.20. func inFramesetIM(p *parser) bool { switch p.tok.Type { case CommentToken: @@ -1720,6 +1923,11 @@ func inFramesetIM(p *parser) bool { p.acknowledgeSelfClosingTag() case a.Noframes: return inHeadIM(p) + case a.Template: + // TODO: remove this divergence from the HTML5 spec. + // + // See https://bugs.chromium.org/p/chromium/issues/detail?id=829668 + return inTemplateIM(p) } case EndTagToken: switch p.tok.DataAtom { @@ -1738,7 +1946,7 @@ func inFramesetIM(p *parser) bool { return true } -// Section 12.2.5.4.20. +// Section 12.2.6.4.21. func afterFramesetIM(p *parser) bool { switch p.tok.Type { case CommentToken: @@ -1777,7 +1985,7 @@ func afterFramesetIM(p *parser) bool { return true } -// Section 12.2.5.4.21. +// Section 12.2.6.4.22. func afterAfterBodyIM(p *parser) bool { switch p.tok.Type { case ErrorToken: @@ -1806,7 +2014,7 @@ func afterAfterBodyIM(p *parser) bool { return false } -// Section 12.2.5.4.22. +// Section 12.2.6.4.23. func afterAfterFramesetIM(p *parser) bool { switch p.tok.Type { case CommentToken: @@ -1844,7 +2052,7 @@ func afterAfterFramesetIM(p *parser) bool { const whitespaceOrNUL = whitespace + "\x00" -// Section 12.2.5.5. +// Section 12.2.6.5 func parseForeignContent(p *parser) bool { switch p.tok.Type { case TextToken: @@ -1924,7 +2132,7 @@ func parseForeignContent(p *parser) bool { return true } -// Section 12.2.5. +// Section 12.2.6. func (p *parser) inForeignContent() bool { if len(p.oe) == 0 { return false @@ -2064,6 +2272,9 @@ func ParseFragment(r io.Reader, context *Node) ([]*Node, error) { } p.doc.AppendChild(root) p.oe = nodeStack{root} + if context != nil && context.DataAtom == a.Template { + p.templateStack = append(p.templateStack, inTemplateIM) + } p.resetInsertionMode() for n := context; n != nil; n = n.Parent { diff --git a/vendor/golang.org/x/net/html/token.go b/vendor/golang.org/x/net/html/token.go index 893e272a..e3c01d7c 100644 --- a/vendor/golang.org/x/net/html/token.go +++ b/vendor/golang.org/x/net/html/token.go @@ -1161,8 +1161,8 @@ func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) { return nil, nil, false } -// Token returns the next Token. The result's Data and Attr values remain valid -// after subsequent Next calls. +// Token returns the current Token. The result's Data and Attr values remain +// valid after subsequent Next calls. func (z *Tokenizer) Token() Token { t := Token{Type: z.tt} switch z.tt { diff --git a/vendor/golang.org/x/net/internal/iana/const.go b/vendor/golang.org/x/net/internal/iana/const.go new file mode 100644 index 00000000..826633e1 --- /dev/null +++ b/vendor/golang.org/x/net/internal/iana/const.go @@ -0,0 +1,227 @@ +// go generate gen.go +// Code generated by the command above; DO NOT EDIT. + +// Package iana provides protocol number resources managed by the Internet Assigned Numbers Authority (IANA). +package iana // import "golang.org/x/net/internal/iana" + +// Differentiated Services Field Codepoints (DSCP), Updated: 2017-05-12 +const ( + DiffServCS0 = 0x0 // CS0 + DiffServCS1 = 0x20 // CS1 + DiffServCS2 = 0x40 // CS2 + DiffServCS3 = 0x60 // CS3 + DiffServCS4 = 0x80 // CS4 + DiffServCS5 = 0xa0 // CS5 + DiffServCS6 = 0xc0 // CS6 + DiffServCS7 = 0xe0 // CS7 + DiffServAF11 = 0x28 // AF11 + DiffServAF12 = 0x30 // AF12 + DiffServAF13 = 0x38 // AF13 + DiffServAF21 = 0x48 // AF21 + DiffServAF22 = 0x50 // AF22 + DiffServAF23 = 0x58 // AF23 + DiffServAF31 = 0x68 // AF31 + DiffServAF32 = 0x70 // AF32 + DiffServAF33 = 0x78 // AF33 + DiffServAF41 = 0x88 // AF41 + DiffServAF42 = 0x90 // AF42 + DiffServAF43 = 0x98 // AF43 + DiffServEF = 0xb8 // EF + DiffServVOICEADMIT = 0xb0 // VOICE-ADMIT +) + +// IPv4 TOS Byte and IPv6 Traffic Class Octet, Updated: 2001-09-06 +const ( + NotECNTransport = 0x0 // Not-ECT (Not ECN-Capable Transport) + ECNTransport1 = 0x1 // ECT(1) (ECN-Capable Transport(1)) + ECNTransport0 = 0x2 // ECT(0) (ECN-Capable Transport(0)) + CongestionExperienced = 0x3 // CE (Congestion Experienced) +) + +// Protocol Numbers, Updated: 2017-10-13 +const ( + ProtocolIP = 0 // IPv4 encapsulation, pseudo protocol number + ProtocolHOPOPT = 0 // IPv6 Hop-by-Hop Option + ProtocolICMP = 1 // Internet Control Message + ProtocolIGMP = 2 // Internet Group Management + ProtocolGGP = 3 // Gateway-to-Gateway + ProtocolIPv4 = 4 // IPv4 encapsulation + ProtocolST = 5 // Stream + ProtocolTCP = 6 // Transmission Control + ProtocolCBT = 7 // CBT + ProtocolEGP = 8 // Exterior Gateway Protocol + ProtocolIGP = 9 // any private interior gateway (used by Cisco for their IGRP) + ProtocolBBNRCCMON = 10 // BBN RCC Monitoring + ProtocolNVPII = 11 // Network Voice Protocol + ProtocolPUP = 12 // PUP + ProtocolEMCON = 14 // EMCON + ProtocolXNET = 15 // Cross Net Debugger + ProtocolCHAOS = 16 // Chaos + ProtocolUDP = 17 // User Datagram + ProtocolMUX = 18 // Multiplexing + ProtocolDCNMEAS = 19 // DCN Measurement Subsystems + ProtocolHMP = 20 // Host Monitoring + ProtocolPRM = 21 // Packet Radio Measurement + ProtocolXNSIDP = 22 // XEROX NS IDP + ProtocolTRUNK1 = 23 // Trunk-1 + ProtocolTRUNK2 = 24 // Trunk-2 + ProtocolLEAF1 = 25 // Leaf-1 + ProtocolLEAF2 = 26 // Leaf-2 + ProtocolRDP = 27 // Reliable Data Protocol + ProtocolIRTP = 28 // Internet Reliable Transaction + ProtocolISOTP4 = 29 // ISO Transport Protocol Class 4 + ProtocolNETBLT = 30 // Bulk Data Transfer Protocol + ProtocolMFENSP = 31 // MFE Network Services Protocol + ProtocolMERITINP = 32 // MERIT Internodal Protocol + ProtocolDCCP = 33 // Datagram Congestion Control Protocol + Protocol3PC = 34 // Third Party Connect Protocol + ProtocolIDPR = 35 // Inter-Domain Policy Routing Protocol + ProtocolXTP = 36 // XTP + ProtocolDDP = 37 // Datagram Delivery Protocol + ProtocolIDPRCMTP = 38 // IDPR Control Message Transport Proto + ProtocolTPPP = 39 // TP++ Transport Protocol + ProtocolIL = 40 // IL Transport Protocol + ProtocolIPv6 = 41 // IPv6 encapsulation + ProtocolSDRP = 42 // Source Demand Routing Protocol + ProtocolIPv6Route = 43 // Routing Header for IPv6 + ProtocolIPv6Frag = 44 // Fragment Header for IPv6 + ProtocolIDRP = 45 // Inter-Domain Routing Protocol + ProtocolRSVP = 46 // Reservation Protocol + ProtocolGRE = 47 // Generic Routing Encapsulation + ProtocolDSR = 48 // Dynamic Source Routing Protocol + ProtocolBNA = 49 // BNA + ProtocolESP = 50 // Encap Security Payload + ProtocolAH = 51 // Authentication Header + ProtocolINLSP = 52 // Integrated Net Layer Security TUBA + ProtocolNARP = 54 // NBMA Address Resolution Protocol + ProtocolMOBILE = 55 // IP Mobility + ProtocolTLSP = 56 // Transport Layer Security Protocol using Kryptonet key management + ProtocolSKIP = 57 // SKIP + ProtocolIPv6ICMP = 58 // ICMP for IPv6 + ProtocolIPv6NoNxt = 59 // No Next Header for IPv6 + ProtocolIPv6Opts = 60 // Destination Options for IPv6 + ProtocolCFTP = 62 // CFTP + ProtocolSATEXPAK = 64 // SATNET and Backroom EXPAK + ProtocolKRYPTOLAN = 65 // Kryptolan + ProtocolRVD = 66 // MIT Remote Virtual Disk Protocol + ProtocolIPPC = 67 // Internet Pluribus Packet Core + ProtocolSATMON = 69 // SATNET Monitoring + ProtocolVISA = 70 // VISA Protocol + ProtocolIPCV = 71 // Internet Packet Core Utility + ProtocolCPNX = 72 // Computer Protocol Network Executive + ProtocolCPHB = 73 // Computer Protocol Heart Beat + ProtocolWSN = 74 // Wang Span Network + ProtocolPVP = 75 // Packet Video Protocol + ProtocolBRSATMON = 76 // Backroom SATNET Monitoring + ProtocolSUNND = 77 // SUN ND PROTOCOL-Temporary + ProtocolWBMON = 78 // WIDEBAND Monitoring + ProtocolWBEXPAK = 79 // WIDEBAND EXPAK + ProtocolISOIP = 80 // ISO Internet Protocol + ProtocolVMTP = 81 // VMTP + ProtocolSECUREVMTP = 82 // SECURE-VMTP + ProtocolVINES = 83 // VINES + ProtocolTTP = 84 // Transaction Transport Protocol + ProtocolIPTM = 84 // Internet Protocol Traffic Manager + ProtocolNSFNETIGP = 85 // NSFNET-IGP + ProtocolDGP = 86 // Dissimilar Gateway Protocol + ProtocolTCF = 87 // TCF + ProtocolEIGRP = 88 // EIGRP + ProtocolOSPFIGP = 89 // OSPFIGP + ProtocolSpriteRPC = 90 // Sprite RPC Protocol + ProtocolLARP = 91 // Locus Address Resolution Protocol + ProtocolMTP = 92 // Multicast Transport Protocol + ProtocolAX25 = 93 // AX.25 Frames + ProtocolIPIP = 94 // IP-within-IP Encapsulation Protocol + ProtocolSCCSP = 96 // Semaphore Communications Sec. Pro. + ProtocolETHERIP = 97 // Ethernet-within-IP Encapsulation + ProtocolENCAP = 98 // Encapsulation Header + ProtocolGMTP = 100 // GMTP + ProtocolIFMP = 101 // Ipsilon Flow Management Protocol + ProtocolPNNI = 102 // PNNI over IP + ProtocolPIM = 103 // Protocol Independent Multicast + ProtocolARIS = 104 // ARIS + ProtocolSCPS = 105 // SCPS + ProtocolQNX = 106 // QNX + ProtocolAN = 107 // Active Networks + ProtocolIPComp = 108 // IP Payload Compression Protocol + ProtocolSNP = 109 // Sitara Networks Protocol + ProtocolCompaqPeer = 110 // Compaq Peer Protocol + ProtocolIPXinIP = 111 // IPX in IP + ProtocolVRRP = 112 // Virtual Router Redundancy Protocol + ProtocolPGM = 113 // PGM Reliable Transport Protocol + ProtocolL2TP = 115 // Layer Two Tunneling Protocol + ProtocolDDX = 116 // D-II Data Exchange (DDX) + ProtocolIATP = 117 // Interactive Agent Transfer Protocol + ProtocolSTP = 118 // Schedule Transfer Protocol + ProtocolSRP = 119 // SpectraLink Radio Protocol + ProtocolUTI = 120 // UTI + ProtocolSMP = 121 // Simple Message Protocol + ProtocolPTP = 123 // Performance Transparency Protocol + ProtocolISIS = 124 // ISIS over IPv4 + ProtocolFIRE = 125 // FIRE + ProtocolCRTP = 126 // Combat Radio Transport Protocol + ProtocolCRUDP = 127 // Combat Radio User Datagram + ProtocolSSCOPMCE = 128 // SSCOPMCE + ProtocolIPLT = 129 // IPLT + ProtocolSPS = 130 // Secure Packet Shield + ProtocolPIPE = 131 // Private IP Encapsulation within IP + ProtocolSCTP = 132 // Stream Control Transmission Protocol + ProtocolFC = 133 // Fibre Channel + ProtocolRSVPE2EIGNORE = 134 // RSVP-E2E-IGNORE + ProtocolMobilityHeader = 135 // Mobility Header + ProtocolUDPLite = 136 // UDPLite + ProtocolMPLSinIP = 137 // MPLS-in-IP + ProtocolMANET = 138 // MANET Protocols + ProtocolHIP = 139 // Host Identity Protocol + ProtocolShim6 = 140 // Shim6 Protocol + ProtocolWESP = 141 // Wrapped Encapsulating Security Payload + ProtocolROHC = 142 // Robust Header Compression + ProtocolReserved = 255 // Reserved +) + +// Address Family Numbers, Updated: 2016-10-25 +const ( + AddrFamilyIPv4 = 1 // IP (IP version 4) + AddrFamilyIPv6 = 2 // IP6 (IP version 6) + AddrFamilyNSAP = 3 // NSAP + AddrFamilyHDLC = 4 // HDLC (8-bit multidrop) + AddrFamilyBBN1822 = 5 // BBN 1822 + AddrFamily802 = 6 // 802 (includes all 802 media plus Ethernet "canonical format") + AddrFamilyE163 = 7 // E.163 + AddrFamilyE164 = 8 // E.164 (SMDS, Frame Relay, ATM) + AddrFamilyF69 = 9 // F.69 (Telex) + AddrFamilyX121 = 10 // X.121 (X.25, Frame Relay) + AddrFamilyIPX = 11 // IPX + AddrFamilyAppletalk = 12 // Appletalk + AddrFamilyDecnetIV = 13 // Decnet IV + AddrFamilyBanyanVines = 14 // Banyan Vines + AddrFamilyE164withSubaddress = 15 // E.164 with NSAP format subaddress + AddrFamilyDNS = 16 // DNS (Domain Name System) + AddrFamilyDistinguishedName = 17 // Distinguished Name + AddrFamilyASNumber = 18 // AS Number + AddrFamilyXTPoverIPv4 = 19 // XTP over IP version 4 + AddrFamilyXTPoverIPv6 = 20 // XTP over IP version 6 + AddrFamilyXTPnativemodeXTP = 21 // XTP native mode XTP + AddrFamilyFibreChannelWorldWidePortName = 22 // Fibre Channel World-Wide Port Name + AddrFamilyFibreChannelWorldWideNodeName = 23 // Fibre Channel World-Wide Node Name + AddrFamilyGWID = 24 // GWID + AddrFamilyL2VPN = 25 // AFI for L2VPN information + AddrFamilyMPLSTPSectionEndpointID = 26 // MPLS-TP Section Endpoint Identifier + AddrFamilyMPLSTPLSPEndpointID = 27 // MPLS-TP LSP Endpoint Identifier + AddrFamilyMPLSTPPseudowireEndpointID = 28 // MPLS-TP Pseudowire Endpoint Identifier + AddrFamilyMTIPv4 = 29 // MT IP: Multi-Topology IP version 4 + AddrFamilyMTIPv6 = 30 // MT IPv6: Multi-Topology IP version 6 + AddrFamilyEIGRPCommonServiceFamily = 16384 // EIGRP Common Service Family + AddrFamilyEIGRPIPv4ServiceFamily = 16385 // EIGRP IPv4 Service Family + AddrFamilyEIGRPIPv6ServiceFamily = 16386 // EIGRP IPv6 Service Family + AddrFamilyLISPCanonicalAddressFormat = 16387 // LISP Canonical Address Format (LCAF) + AddrFamilyBGPLS = 16388 // BGP-LS + AddrFamily48bitMAC = 16389 // 48-bit MAC + AddrFamily64bitMAC = 16390 // 64-bit MAC + AddrFamilyOUI = 16391 // OUI + AddrFamilyMACFinal24bits = 16392 // MAC/24 + AddrFamilyMACFinal40bits = 16393 // MAC/40 + AddrFamilyIPv6Initial64bits = 16394 // IPv6/64 + AddrFamilyRBridgePortID = 16395 // RBridge Port ID + AddrFamilyTRILLNickname = 16396 // TRILL Nickname +) diff --git a/vendor/golang.org/x/net/internal/iana/gen.go b/vendor/golang.org/x/net/internal/iana/gen.go new file mode 100644 index 00000000..2227e09e --- /dev/null +++ b/vendor/golang.org/x/net/internal/iana/gen.go @@ -0,0 +1,387 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +//go:generate go run gen.go + +// This program generates internet protocol constants and tables by +// reading IANA protocol registries. +package main + +import ( + "bytes" + "encoding/xml" + "fmt" + "go/format" + "io" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" +) + +var registries = []struct { + url string + parse func(io.Writer, io.Reader) error +}{ + { + "https://www.iana.org/assignments/dscp-registry/dscp-registry.xml", + parseDSCPRegistry, + }, + { + "https://www.iana.org/assignments/ipv4-tos-byte/ipv4-tos-byte.xml", + parseTOSTCByte, + }, + { + "https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xml", + parseProtocolNumbers, + }, + { + "http://www.iana.org/assignments/address-family-numbers/address-family-numbers.xml", + parseAddrFamilyNumbers, + }, +} + +func main() { + var bb bytes.Buffer + fmt.Fprintf(&bb, "// go generate gen.go\n") + fmt.Fprintf(&bb, "// Code generated by the command above; DO NOT EDIT.\n\n") + fmt.Fprintf(&bb, "// Package iana provides protocol number resources managed by the Internet Assigned Numbers Authority (IANA).\n") + fmt.Fprintf(&bb, `package iana // import "golang.org/x/net/internal/iana"`+"\n\n") + for _, r := range registries { + resp, err := http.Get(r.url) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "got HTTP status code %v for %v\n", resp.StatusCode, r.url) + os.Exit(1) + } + if err := r.parse(&bb, resp.Body); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Fprintf(&bb, "\n") + } + b, err := format.Source(bb.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := ioutil.WriteFile("const.go", b, 0644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func parseDSCPRegistry(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var dr dscpRegistry + if err := dec.Decode(&dr); err != nil { + return err + } + drs := dr.escape() + fmt.Fprintf(w, "// %s, Updated: %s\n", dr.Title, dr.Updated) + fmt.Fprintf(w, "const (\n") + for _, dr := range drs { + fmt.Fprintf(w, "DiffServ%s = %#x", dr.Name, dr.Value) + fmt.Fprintf(w, "// %s\n", dr.OrigName) + } + fmt.Fprintf(w, ")\n") + return nil +} + +type dscpRegistry struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + Note string `xml:"note"` + RegTitle string `xml:"registry>title"` + PoolRecords []struct { + Name string `xml:"name"` + Space string `xml:"space"` + } `xml:"registry>record"` + Records []struct { + Name string `xml:"name"` + Space string `xml:"space"` + } `xml:"registry>registry>record"` +} + +type canonDSCPRecord struct { + OrigName string + Name string + Value int +} + +func (drr *dscpRegistry) escape() []canonDSCPRecord { + drs := make([]canonDSCPRecord, len(drr.Records)) + sr := strings.NewReplacer( + "+", "", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, dr := range drr.Records { + s := strings.TrimSpace(dr.Name) + drs[i].OrigName = s + drs[i].Name = sr.Replace(s) + n, err := strconv.ParseUint(dr.Space, 2, 8) + if err != nil { + continue + } + drs[i].Value = int(n) << 2 + } + return drs +} + +func parseTOSTCByte(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var ttb tosTCByte + if err := dec.Decode(&ttb); err != nil { + return err + } + trs := ttb.escape() + fmt.Fprintf(w, "// %s, Updated: %s\n", ttb.Title, ttb.Updated) + fmt.Fprintf(w, "const (\n") + for _, tr := range trs { + fmt.Fprintf(w, "%s = %#x", tr.Keyword, tr.Value) + fmt.Fprintf(w, "// %s\n", tr.OrigKeyword) + } + fmt.Fprintf(w, ")\n") + return nil +} + +type tosTCByte struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + Note string `xml:"note"` + RegTitle string `xml:"registry>title"` + Records []struct { + Binary string `xml:"binary"` + Keyword string `xml:"keyword"` + } `xml:"registry>record"` +} + +type canonTOSTCByteRecord struct { + OrigKeyword string + Keyword string + Value int +} + +func (ttb *tosTCByte) escape() []canonTOSTCByteRecord { + trs := make([]canonTOSTCByteRecord, len(ttb.Records)) + sr := strings.NewReplacer( + "Capable", "", + "(", "", + ")", "", + "+", "", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, tr := range ttb.Records { + s := strings.TrimSpace(tr.Keyword) + trs[i].OrigKeyword = s + ss := strings.Split(s, " ") + if len(ss) > 1 { + trs[i].Keyword = strings.Join(ss[1:], " ") + } else { + trs[i].Keyword = ss[0] + } + trs[i].Keyword = sr.Replace(trs[i].Keyword) + n, err := strconv.ParseUint(tr.Binary, 2, 8) + if err != nil { + continue + } + trs[i].Value = int(n) + } + return trs +} + +func parseProtocolNumbers(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var pn protocolNumbers + if err := dec.Decode(&pn); err != nil { + return err + } + prs := pn.escape() + prs = append([]canonProtocolRecord{{ + Name: "IP", + Descr: "IPv4 encapsulation, pseudo protocol number", + Value: 0, + }}, prs...) + fmt.Fprintf(w, "// %s, Updated: %s\n", pn.Title, pn.Updated) + fmt.Fprintf(w, "const (\n") + for _, pr := range prs { + if pr.Name == "" { + continue + } + fmt.Fprintf(w, "Protocol%s = %d", pr.Name, pr.Value) + s := pr.Descr + if s == "" { + s = pr.OrigName + } + fmt.Fprintf(w, "// %s\n", s) + } + fmt.Fprintf(w, ")\n") + return nil +} + +type protocolNumbers struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + RegTitle string `xml:"registry>title"` + Note string `xml:"registry>note"` + Records []struct { + Value string `xml:"value"` + Name string `xml:"name"` + Descr string `xml:"description"` + } `xml:"registry>record"` +} + +type canonProtocolRecord struct { + OrigName string + Name string + Descr string + Value int +} + +func (pn *protocolNumbers) escape() []canonProtocolRecord { + prs := make([]canonProtocolRecord, len(pn.Records)) + sr := strings.NewReplacer( + "-in-", "in", + "-within-", "within", + "-over-", "over", + "+", "P", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, pr := range pn.Records { + if strings.Contains(pr.Name, "Deprecated") || + strings.Contains(pr.Name, "deprecated") { + continue + } + prs[i].OrigName = pr.Name + s := strings.TrimSpace(pr.Name) + switch pr.Name { + case "ISIS over IPv4": + prs[i].Name = "ISIS" + case "manet": + prs[i].Name = "MANET" + default: + prs[i].Name = sr.Replace(s) + } + ss := strings.Split(pr.Descr, "\n") + for i := range ss { + ss[i] = strings.TrimSpace(ss[i]) + } + if len(ss) > 1 { + prs[i].Descr = strings.Join(ss, " ") + } else { + prs[i].Descr = ss[0] + } + prs[i].Value, _ = strconv.Atoi(pr.Value) + } + return prs +} + +func parseAddrFamilyNumbers(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var afn addrFamilylNumbers + if err := dec.Decode(&afn); err != nil { + return err + } + afrs := afn.escape() + fmt.Fprintf(w, "// %s, Updated: %s\n", afn.Title, afn.Updated) + fmt.Fprintf(w, "const (\n") + for _, afr := range afrs { + if afr.Name == "" { + continue + } + fmt.Fprintf(w, "AddrFamily%s = %d", afr.Name, afr.Value) + fmt.Fprintf(w, "// %s\n", afr.Descr) + } + fmt.Fprintf(w, ")\n") + return nil +} + +type addrFamilylNumbers struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + RegTitle string `xml:"registry>title"` + Note string `xml:"registry>note"` + Records []struct { + Value string `xml:"value"` + Descr string `xml:"description"` + } `xml:"registry>record"` +} + +type canonAddrFamilyRecord struct { + Name string + Descr string + Value int +} + +func (afn *addrFamilylNumbers) escape() []canonAddrFamilyRecord { + afrs := make([]canonAddrFamilyRecord, len(afn.Records)) + sr := strings.NewReplacer( + "IP version 4", "IPv4", + "IP version 6", "IPv6", + "Identifier", "ID", + "-", "", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, afr := range afn.Records { + if strings.Contains(afr.Descr, "Unassigned") || + strings.Contains(afr.Descr, "Reserved") { + continue + } + afrs[i].Descr = afr.Descr + s := strings.TrimSpace(afr.Descr) + switch s { + case "IP (IP version 4)": + afrs[i].Name = "IPv4" + case "IP6 (IP version 6)": + afrs[i].Name = "IPv6" + case "AFI for L2VPN information": + afrs[i].Name = "L2VPN" + case "E.164 with NSAP format subaddress": + afrs[i].Name = "E164withSubaddress" + case "MT IP: Multi-Topology IP version 4": + afrs[i].Name = "MTIPv4" + case "MAC/24": + afrs[i].Name = "MACFinal24bits" + case "MAC/40": + afrs[i].Name = "MACFinal40bits" + case "IPv6/64": + afrs[i].Name = "IPv6Initial64bits" + default: + n := strings.Index(s, "(") + if n > 0 { + s = s[:n] + } + n = strings.Index(s, ":") + if n > 0 { + s = s[:n] + } + afrs[i].Name = sr.Replace(s) + } + afrs[i].Value, _ = strconv.Atoi(afr.Value) + } + return afrs +} diff --git a/vendor/golang.org/x/net/internal/socket/cmsghdr.go b/vendor/golang.org/x/net/internal/socket/cmsghdr.go new file mode 100644 index 00000000..1eb07d26 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/cmsghdr.go @@ -0,0 +1,11 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package socket + +func (h *cmsghdr) len() int { return int(h.Len) } +func (h *cmsghdr) lvl() int { return int(h.Level) } +func (h *cmsghdr) typ() int { return int(h.Type) } diff --git a/vendor/golang.org/x/net/internal/socket/cmsghdr_bsd.go b/vendor/golang.org/x/net/internal/socket/cmsghdr_bsd.go new file mode 100644 index 00000000..d1d0c2de --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/cmsghdr_bsd.go @@ -0,0 +1,13 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd + +package socket + +func (h *cmsghdr) set(l, lvl, typ int) { + h.Len = uint32(l) + h.Level = int32(lvl) + h.Type = int32(typ) +} diff --git a/vendor/golang.org/x/net/internal/socket/cmsghdr_linux_32bit.go b/vendor/golang.org/x/net/internal/socket/cmsghdr_linux_32bit.go new file mode 100644 index 00000000..bac66811 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/cmsghdr_linux_32bit.go @@ -0,0 +1,14 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm mips mipsle 386 +// +build linux + +package socket + +func (h *cmsghdr) set(l, lvl, typ int) { + h.Len = uint32(l) + h.Level = int32(lvl) + h.Type = int32(typ) +} diff --git a/vendor/golang.org/x/net/internal/socket/cmsghdr_linux_64bit.go b/vendor/golang.org/x/net/internal/socket/cmsghdr_linux_64bit.go new file mode 100644 index 00000000..63f0534f --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/cmsghdr_linux_64bit.go @@ -0,0 +1,14 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 amd64 ppc64 ppc64le mips64 mips64le s390x +// +build linux + +package socket + +func (h *cmsghdr) set(l, lvl, typ int) { + h.Len = uint64(l) + h.Level = int32(lvl) + h.Type = int32(typ) +} diff --git a/vendor/golang.org/x/net/internal/socket/cmsghdr_solaris_64bit.go b/vendor/golang.org/x/net/internal/socket/cmsghdr_solaris_64bit.go new file mode 100644 index 00000000..7dedd430 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/cmsghdr_solaris_64bit.go @@ -0,0 +1,14 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build amd64 +// +build solaris + +package socket + +func (h *cmsghdr) set(l, lvl, typ int) { + h.Len = uint32(l) + h.Level = int32(lvl) + h.Type = int32(typ) +} diff --git a/vendor/golang.org/x/net/internal/socket/cmsghdr_stub.go b/vendor/golang.org/x/net/internal/socket/cmsghdr_stub.go new file mode 100644 index 00000000..a4e71226 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/cmsghdr_stub.go @@ -0,0 +1,17 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris + +package socket + +type cmsghdr struct{} + +const sizeofCmsghdr = 0 + +func (h *cmsghdr) len() int { return 0 } +func (h *cmsghdr) lvl() int { return 0 } +func (h *cmsghdr) typ() int { return 0 } + +func (h *cmsghdr) set(l, lvl, typ int) {} diff --git a/vendor/golang.org/x/net/internal/socket/defs_darwin.go b/vendor/golang.org/x/net/internal/socket/defs_darwin.go new file mode 100644 index 00000000..14e28c0b --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_darwin.go @@ -0,0 +1,44 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +const ( + sysAF_UNSPEC = C.AF_UNSPEC + sysAF_INET = C.AF_INET + sysAF_INET6 = C.AF_INET6 + + sysSOCK_RAW = C.SOCK_RAW +) + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_dragonfly.go b/vendor/golang.org/x/net/internal/socket/defs_dragonfly.go new file mode 100644 index 00000000..14e28c0b --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_dragonfly.go @@ -0,0 +1,44 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +const ( + sysAF_UNSPEC = C.AF_UNSPEC + sysAF_INET = C.AF_INET + sysAF_INET6 = C.AF_INET6 + + sysSOCK_RAW = C.SOCK_RAW +) + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_freebsd.go b/vendor/golang.org/x/net/internal/socket/defs_freebsd.go new file mode 100644 index 00000000..14e28c0b --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_freebsd.go @@ -0,0 +1,44 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +const ( + sysAF_UNSPEC = C.AF_UNSPEC + sysAF_INET = C.AF_INET + sysAF_INET6 = C.AF_INET6 + + sysSOCK_RAW = C.SOCK_RAW +) + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_linux.go b/vendor/golang.org/x/net/internal/socket/defs_linux.go new file mode 100644 index 00000000..ce9ec2f6 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_linux.go @@ -0,0 +1,49 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include +#include + +#define _GNU_SOURCE +#include +*/ +import "C" + +const ( + sysAF_UNSPEC = C.AF_UNSPEC + sysAF_INET = C.AF_INET + sysAF_INET6 = C.AF_INET6 + + sysSOCK_RAW = C.SOCK_RAW +) + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type mmsghdr C.struct_mmsghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofMmsghdr = C.sizeof_struct_mmsghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_netbsd.go b/vendor/golang.org/x/net/internal/socket/defs_netbsd.go new file mode 100644 index 00000000..3f843356 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_netbsd.go @@ -0,0 +1,47 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +const ( + sysAF_UNSPEC = C.AF_UNSPEC + sysAF_INET = C.AF_INET + sysAF_INET6 = C.AF_INET6 + + sysSOCK_RAW = C.SOCK_RAW +) + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type mmsghdr C.struct_mmsghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofMmsghdr = C.sizeof_struct_mmsghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_openbsd.go b/vendor/golang.org/x/net/internal/socket/defs_openbsd.go new file mode 100644 index 00000000..14e28c0b --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_openbsd.go @@ -0,0 +1,44 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +const ( + sysAF_UNSPEC = C.AF_UNSPEC + sysAF_INET = C.AF_INET + sysAF_INET6 = C.AF_INET6 + + sysSOCK_RAW = C.SOCK_RAW +) + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_solaris.go b/vendor/golang.org/x/net/internal/socket/defs_solaris.go new file mode 100644 index 00000000..14e28c0b --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_solaris.go @@ -0,0 +1,44 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +const ( + sysAF_UNSPEC = C.AF_UNSPEC + sysAF_INET = C.AF_INET + sysAF_INET6 = C.AF_INET6 + + sysSOCK_RAW = C.SOCK_RAW +) + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/error_unix.go b/vendor/golang.org/x/net/internal/socket/error_unix.go new file mode 100644 index 00000000..93dff918 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/error_unix.go @@ -0,0 +1,31 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package socket + +import "syscall" + +var ( + errEAGAIN error = syscall.EAGAIN + errEINVAL error = syscall.EINVAL + errENOENT error = syscall.ENOENT +) + +// errnoErr returns common boxed Errno values, to prevent allocations +// at runtime. +func errnoErr(errno syscall.Errno) error { + switch errno { + case 0: + return nil + case syscall.EAGAIN: + return errEAGAIN + case syscall.EINVAL: + return errEINVAL + case syscall.ENOENT: + return errENOENT + } + return errno +} diff --git a/vendor/golang.org/x/net/internal/socket/error_windows.go b/vendor/golang.org/x/net/internal/socket/error_windows.go new file mode 100644 index 00000000..6a6379a8 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/error_windows.go @@ -0,0 +1,26 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +import "syscall" + +var ( + errERROR_IO_PENDING error = syscall.ERROR_IO_PENDING + errEINVAL error = syscall.EINVAL +) + +// errnoErr returns common boxed Errno values, to prevent allocations +// at runtime. +func errnoErr(errno syscall.Errno) error { + switch errno { + case 0: + return nil + case syscall.ERROR_IO_PENDING: + return errERROR_IO_PENDING + case syscall.EINVAL: + return errEINVAL + } + return errno +} diff --git a/vendor/golang.org/x/net/internal/socket/iovec_32bit.go b/vendor/golang.org/x/net/internal/socket/iovec_32bit.go new file mode 100644 index 00000000..05d6082d --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/iovec_32bit.go @@ -0,0 +1,19 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm mips mipsle 386 +// +build darwin dragonfly freebsd linux netbsd openbsd + +package socket + +import "unsafe" + +func (v *iovec) set(b []byte) { + l := len(b) + if l == 0 { + return + } + v.Base = (*byte)(unsafe.Pointer(&b[0])) + v.Len = uint32(l) +} diff --git a/vendor/golang.org/x/net/internal/socket/iovec_64bit.go b/vendor/golang.org/x/net/internal/socket/iovec_64bit.go new file mode 100644 index 00000000..afb34ad5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/iovec_64bit.go @@ -0,0 +1,19 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 amd64 ppc64 ppc64le mips64 mips64le s390x +// +build darwin dragonfly freebsd linux netbsd openbsd + +package socket + +import "unsafe" + +func (v *iovec) set(b []byte) { + l := len(b) + if l == 0 { + return + } + v.Base = (*byte)(unsafe.Pointer(&b[0])) + v.Len = uint64(l) +} diff --git a/vendor/golang.org/x/net/internal/socket/iovec_solaris_64bit.go b/vendor/golang.org/x/net/internal/socket/iovec_solaris_64bit.go new file mode 100644 index 00000000..8d17a40c --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/iovec_solaris_64bit.go @@ -0,0 +1,19 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build amd64 +// +build solaris + +package socket + +import "unsafe" + +func (v *iovec) set(b []byte) { + l := len(b) + if l == 0 { + return + } + v.Base = (*int8)(unsafe.Pointer(&b[0])) + v.Len = uint64(l) +} diff --git a/vendor/golang.org/x/net/internal/socket/iovec_stub.go b/vendor/golang.org/x/net/internal/socket/iovec_stub.go new file mode 100644 index 00000000..c87d2a93 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/iovec_stub.go @@ -0,0 +1,11 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris + +package socket + +type iovec struct{} + +func (v *iovec) set(b []byte) {} diff --git a/vendor/golang.org/x/net/internal/socket/mmsghdr_stub.go b/vendor/golang.org/x/net/internal/socket/mmsghdr_stub.go new file mode 100644 index 00000000..2e80a9cb --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/mmsghdr_stub.go @@ -0,0 +1,21 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux,!netbsd + +package socket + +import "net" + +type mmsghdr struct{} + +type mmsghdrs []mmsghdr + +func (hs mmsghdrs) pack(ms []Message, parseFn func([]byte, string) (net.Addr, error), marshalFn func(net.Addr) []byte) error { + return nil +} + +func (hs mmsghdrs) unpack(ms []Message, parseFn func([]byte, string) (net.Addr, error), hint string) error { + return nil +} diff --git a/vendor/golang.org/x/net/internal/socket/mmsghdr_unix.go b/vendor/golang.org/x/net/internal/socket/mmsghdr_unix.go new file mode 100644 index 00000000..3c42ea7a --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/mmsghdr_unix.go @@ -0,0 +1,42 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux netbsd + +package socket + +import "net" + +type mmsghdrs []mmsghdr + +func (hs mmsghdrs) pack(ms []Message, parseFn func([]byte, string) (net.Addr, error), marshalFn func(net.Addr) []byte) error { + for i := range hs { + vs := make([]iovec, len(ms[i].Buffers)) + var sa []byte + if parseFn != nil { + sa = make([]byte, sizeofSockaddrInet6) + } + if marshalFn != nil { + sa = marshalFn(ms[i].Addr) + } + hs[i].Hdr.pack(vs, ms[i].Buffers, ms[i].OOB, sa) + } + return nil +} + +func (hs mmsghdrs) unpack(ms []Message, parseFn func([]byte, string) (net.Addr, error), hint string) error { + for i := range hs { + ms[i].N = int(hs[i].Len) + ms[i].NN = hs[i].Hdr.controllen() + ms[i].Flags = hs[i].Hdr.flags() + if parseFn != nil { + var err error + ms[i].Addr, err = parseFn(hs[i].Hdr.name(), hint) + if err != nil { + return err + } + } + } + return nil +} diff --git a/vendor/golang.org/x/net/internal/socket/msghdr_bsd.go b/vendor/golang.org/x/net/internal/socket/msghdr_bsd.go new file mode 100644 index 00000000..5567afc8 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/msghdr_bsd.go @@ -0,0 +1,39 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd + +package socket + +import "unsafe" + +func (h *msghdr) pack(vs []iovec, bs [][]byte, oob []byte, sa []byte) { + for i := range vs { + vs[i].set(bs[i]) + } + h.setIov(vs) + if len(oob) > 0 { + h.Control = (*byte)(unsafe.Pointer(&oob[0])) + h.Controllen = uint32(len(oob)) + } + if sa != nil { + h.Name = (*byte)(unsafe.Pointer(&sa[0])) + h.Namelen = uint32(len(sa)) + } +} + +func (h *msghdr) name() []byte { + if h.Name != nil && h.Namelen > 0 { + return (*[sizeofSockaddrInet6]byte)(unsafe.Pointer(h.Name))[:h.Namelen] + } + return nil +} + +func (h *msghdr) controllen() int { + return int(h.Controllen) +} + +func (h *msghdr) flags() int { + return int(h.Flags) +} diff --git a/vendor/golang.org/x/net/internal/socket/msghdr_bsdvar.go b/vendor/golang.org/x/net/internal/socket/msghdr_bsdvar.go new file mode 100644 index 00000000..b8c87b72 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/msghdr_bsdvar.go @@ -0,0 +1,16 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd + +package socket + +func (h *msghdr) setIov(vs []iovec) { + l := len(vs) + if l == 0 { + return + } + h.Iov = &vs[0] + h.Iovlen = int32(l) +} diff --git a/vendor/golang.org/x/net/internal/socket/msghdr_linux.go b/vendor/golang.org/x/net/internal/socket/msghdr_linux.go new file mode 100644 index 00000000..5a38798c --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/msghdr_linux.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +import "unsafe" + +func (h *msghdr) pack(vs []iovec, bs [][]byte, oob []byte, sa []byte) { + for i := range vs { + vs[i].set(bs[i]) + } + h.setIov(vs) + if len(oob) > 0 { + h.setControl(oob) + } + if sa != nil { + h.Name = (*byte)(unsafe.Pointer(&sa[0])) + h.Namelen = uint32(len(sa)) + } +} + +func (h *msghdr) name() []byte { + if h.Name != nil && h.Namelen > 0 { + return (*[sizeofSockaddrInet6]byte)(unsafe.Pointer(h.Name))[:h.Namelen] + } + return nil +} + +func (h *msghdr) controllen() int { + return int(h.Controllen) +} + +func (h *msghdr) flags() int { + return int(h.Flags) +} diff --git a/vendor/golang.org/x/net/internal/socket/msghdr_linux_32bit.go b/vendor/golang.org/x/net/internal/socket/msghdr_linux_32bit.go new file mode 100644 index 00000000..a7a5987c --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/msghdr_linux_32bit.go @@ -0,0 +1,24 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm mips mipsle 386 +// +build linux + +package socket + +import "unsafe" + +func (h *msghdr) setIov(vs []iovec) { + l := len(vs) + if l == 0 { + return + } + h.Iov = &vs[0] + h.Iovlen = uint32(l) +} + +func (h *msghdr) setControl(b []byte) { + h.Control = (*byte)(unsafe.Pointer(&b[0])) + h.Controllen = uint32(len(b)) +} diff --git a/vendor/golang.org/x/net/internal/socket/msghdr_linux_64bit.go b/vendor/golang.org/x/net/internal/socket/msghdr_linux_64bit.go new file mode 100644 index 00000000..610fc4f3 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/msghdr_linux_64bit.go @@ -0,0 +1,24 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 amd64 ppc64 ppc64le mips64 mips64le s390x +// +build linux + +package socket + +import "unsafe" + +func (h *msghdr) setIov(vs []iovec) { + l := len(vs) + if l == 0 { + return + } + h.Iov = &vs[0] + h.Iovlen = uint64(l) +} + +func (h *msghdr) setControl(b []byte) { + h.Control = (*byte)(unsafe.Pointer(&b[0])) + h.Controllen = uint64(len(b)) +} diff --git a/vendor/golang.org/x/net/internal/socket/msghdr_openbsd.go b/vendor/golang.org/x/net/internal/socket/msghdr_openbsd.go new file mode 100644 index 00000000..71a69e25 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/msghdr_openbsd.go @@ -0,0 +1,14 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +func (h *msghdr) setIov(vs []iovec) { + l := len(vs) + if l == 0 { + return + } + h.Iov = &vs[0] + h.Iovlen = uint32(l) +} diff --git a/vendor/golang.org/x/net/internal/socket/msghdr_solaris_64bit.go b/vendor/golang.org/x/net/internal/socket/msghdr_solaris_64bit.go new file mode 100644 index 00000000..6465b207 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/msghdr_solaris_64bit.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build amd64 +// +build solaris + +package socket + +import "unsafe" + +func (h *msghdr) pack(vs []iovec, bs [][]byte, oob []byte, sa []byte) { + for i := range vs { + vs[i].set(bs[i]) + } + if len(vs) > 0 { + h.Iov = &vs[0] + h.Iovlen = int32(len(vs)) + } + if len(oob) > 0 { + h.Accrights = (*int8)(unsafe.Pointer(&oob[0])) + h.Accrightslen = int32(len(oob)) + } + if sa != nil { + h.Name = (*byte)(unsafe.Pointer(&sa[0])) + h.Namelen = uint32(len(sa)) + } +} + +func (h *msghdr) controllen() int { + return int(h.Accrightslen) +} + +func (h *msghdr) flags() int { + return int(NativeEndian.Uint32(h.Pad_cgo_2[:])) +} diff --git a/vendor/golang.org/x/net/internal/socket/msghdr_stub.go b/vendor/golang.org/x/net/internal/socket/msghdr_stub.go new file mode 100644 index 00000000..64e81733 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/msghdr_stub.go @@ -0,0 +1,14 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris + +package socket + +type msghdr struct{} + +func (h *msghdr) pack(vs []iovec, bs [][]byte, oob []byte, sa []byte) {} +func (h *msghdr) name() []byte { return nil } +func (h *msghdr) controllen() int { return 0 } +func (h *msghdr) flags() int { return 0 } diff --git a/vendor/golang.org/x/net/internal/socket/rawconn.go b/vendor/golang.org/x/net/internal/socket/rawconn.go new file mode 100644 index 00000000..d6871d55 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/rawconn.go @@ -0,0 +1,66 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +package socket + +import ( + "errors" + "net" + "os" + "syscall" +) + +// A Conn represents a raw connection. +type Conn struct { + network string + c syscall.RawConn +} + +// NewConn returns a new raw connection. +func NewConn(c net.Conn) (*Conn, error) { + var err error + var cc Conn + switch c := c.(type) { + case *net.TCPConn: + cc.network = "tcp" + cc.c, err = c.SyscallConn() + case *net.UDPConn: + cc.network = "udp" + cc.c, err = c.SyscallConn() + case *net.IPConn: + cc.network = "ip" + cc.c, err = c.SyscallConn() + default: + return nil, errors.New("unknown connection type") + } + if err != nil { + return nil, err + } + return &cc, nil +} + +func (o *Option) get(c *Conn, b []byte) (int, error) { + var operr error + var n int + fn := func(s uintptr) { + n, operr = getsockopt(s, o.Level, o.Name, b) + } + if err := c.c.Control(fn); err != nil { + return 0, err + } + return n, os.NewSyscallError("getsockopt", operr) +} + +func (o *Option) set(c *Conn, b []byte) error { + var operr error + fn := func(s uintptr) { + operr = setsockopt(s, o.Level, o.Name, b) + } + if err := c.c.Control(fn); err != nil { + return err + } + return os.NewSyscallError("setsockopt", operr) +} diff --git a/vendor/golang.org/x/net/internal/socket/rawconn_mmsg.go b/vendor/golang.org/x/net/internal/socket/rawconn_mmsg.go new file mode 100644 index 00000000..499164a3 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/rawconn_mmsg.go @@ -0,0 +1,74 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 +// +build linux + +package socket + +import ( + "net" + "os" + "syscall" +) + +func (c *Conn) recvMsgs(ms []Message, flags int) (int, error) { + hs := make(mmsghdrs, len(ms)) + var parseFn func([]byte, string) (net.Addr, error) + if c.network != "tcp" { + parseFn = parseInetAddr + } + if err := hs.pack(ms, parseFn, nil); err != nil { + return 0, err + } + var operr error + var n int + fn := func(s uintptr) bool { + n, operr = recvmmsg(s, hs, flags) + if operr == syscall.EAGAIN { + return false + } + return true + } + if err := c.c.Read(fn); err != nil { + return n, err + } + if operr != nil { + return n, os.NewSyscallError("recvmmsg", operr) + } + if err := hs[:n].unpack(ms[:n], parseFn, c.network); err != nil { + return n, err + } + return n, nil +} + +func (c *Conn) sendMsgs(ms []Message, flags int) (int, error) { + hs := make(mmsghdrs, len(ms)) + var marshalFn func(net.Addr) []byte + if c.network != "tcp" { + marshalFn = marshalInetAddr + } + if err := hs.pack(ms, nil, marshalFn); err != nil { + return 0, err + } + var operr error + var n int + fn := func(s uintptr) bool { + n, operr = sendmmsg(s, hs, flags) + if operr == syscall.EAGAIN { + return false + } + return true + } + if err := c.c.Write(fn); err != nil { + return n, err + } + if operr != nil { + return n, os.NewSyscallError("sendmmsg", operr) + } + if err := hs[:n].unpack(ms[:n], nil, ""); err != nil { + return n, err + } + return n, nil +} diff --git a/vendor/golang.org/x/net/internal/socket/rawconn_msg.go b/vendor/golang.org/x/net/internal/socket/rawconn_msg.go new file mode 100644 index 00000000..b21d2e64 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/rawconn_msg.go @@ -0,0 +1,77 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 +// +build darwin dragonfly freebsd linux netbsd openbsd solaris windows + +package socket + +import ( + "os" + "syscall" +) + +func (c *Conn) recvMsg(m *Message, flags int) error { + var h msghdr + vs := make([]iovec, len(m.Buffers)) + var sa []byte + if c.network != "tcp" { + sa = make([]byte, sizeofSockaddrInet6) + } + h.pack(vs, m.Buffers, m.OOB, sa) + var operr error + var n int + fn := func(s uintptr) bool { + n, operr = recvmsg(s, &h, flags) + if operr == syscall.EAGAIN { + return false + } + return true + } + if err := c.c.Read(fn); err != nil { + return err + } + if operr != nil { + return os.NewSyscallError("recvmsg", operr) + } + if c.network != "tcp" { + var err error + m.Addr, err = parseInetAddr(sa[:], c.network) + if err != nil { + return err + } + } + m.N = n + m.NN = h.controllen() + m.Flags = h.flags() + return nil +} + +func (c *Conn) sendMsg(m *Message, flags int) error { + var h msghdr + vs := make([]iovec, len(m.Buffers)) + var sa []byte + if m.Addr != nil { + sa = marshalInetAddr(m.Addr) + } + h.pack(vs, m.Buffers, m.OOB, sa) + var operr error + var n int + fn := func(s uintptr) bool { + n, operr = sendmsg(s, &h, flags) + if operr == syscall.EAGAIN { + return false + } + return true + } + if err := c.c.Write(fn); err != nil { + return err + } + if operr != nil { + return os.NewSyscallError("sendmsg", operr) + } + m.N = n + m.NN = len(m.OOB) + return nil +} diff --git a/vendor/golang.org/x/net/internal/socket/rawconn_nommsg.go b/vendor/golang.org/x/net/internal/socket/rawconn_nommsg.go new file mode 100644 index 00000000..f78832aa --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/rawconn_nommsg.go @@ -0,0 +1,18 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 +// +build !linux + +package socket + +import "errors" + +func (c *Conn) recvMsgs(ms []Message, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func (c *Conn) sendMsgs(ms []Message, flags int) (int, error) { + return 0, errors.New("not implemented") +} diff --git a/vendor/golang.org/x/net/internal/socket/rawconn_nomsg.go b/vendor/golang.org/x/net/internal/socket/rawconn_nomsg.go new file mode 100644 index 00000000..96733cbe --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/rawconn_nomsg.go @@ -0,0 +1,18 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package socket + +import "errors" + +func (c *Conn) recvMsg(m *Message, flags int) error { + return errors.New("not implemented") +} + +func (c *Conn) sendMsg(m *Message, flags int) error { + return errors.New("not implemented") +} diff --git a/vendor/golang.org/x/net/internal/socket/rawconn_stub.go b/vendor/golang.org/x/net/internal/socket/rawconn_stub.go new file mode 100644 index 00000000..d2add1a0 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/rawconn_stub.go @@ -0,0 +1,25 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.9 + +package socket + +import "errors" + +func (c *Conn) recvMsg(m *Message, flags int) error { + return errors.New("not implemented") +} + +func (c *Conn) sendMsg(m *Message, flags int) error { + return errors.New("not implemented") +} + +func (c *Conn) recvMsgs(ms []Message, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func (c *Conn) sendMsgs(ms []Message, flags int) (int, error) { + return 0, errors.New("not implemented") +} diff --git a/vendor/golang.org/x/net/internal/socket/reflect.go b/vendor/golang.org/x/net/internal/socket/reflect.go new file mode 100644 index 00000000..bb179f11 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/reflect.go @@ -0,0 +1,62 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.9 + +package socket + +import ( + "errors" + "net" + "os" + "reflect" + "runtime" +) + +// A Conn represents a raw connection. +type Conn struct { + c net.Conn +} + +// NewConn returns a new raw connection. +func NewConn(c net.Conn) (*Conn, error) { + return &Conn{c: c}, nil +} + +func (o *Option) get(c *Conn, b []byte) (int, error) { + s, err := socketOf(c.c) + if err != nil { + return 0, err + } + n, err := getsockopt(s, o.Level, o.Name, b) + return n, os.NewSyscallError("getsockopt", err) +} + +func (o *Option) set(c *Conn, b []byte) error { + s, err := socketOf(c.c) + if err != nil { + return err + } + return os.NewSyscallError("setsockopt", setsockopt(s, o.Level, o.Name, b)) +} + +func socketOf(c net.Conn) (uintptr, error) { + switch c.(type) { + case *net.TCPConn, *net.UDPConn, *net.IPConn: + v := reflect.ValueOf(c) + switch e := v.Elem(); e.Kind() { + case reflect.Struct: + fd := e.FieldByName("conn").FieldByName("fd") + switch e := fd.Elem(); e.Kind() { + case reflect.Struct: + sysfd := e.FieldByName("sysfd") + if runtime.GOOS == "windows" { + return uintptr(sysfd.Uint()), nil + } + return uintptr(sysfd.Int()), nil + } + } + } + return 0, errors.New("invalid type") +} diff --git a/vendor/golang.org/x/net/internal/socket/socket.go b/vendor/golang.org/x/net/internal/socket/socket.go new file mode 100644 index 00000000..5f9730e6 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/socket.go @@ -0,0 +1,285 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package socket provides a portable interface for socket system +// calls. +package socket // import "golang.org/x/net/internal/socket" + +import ( + "errors" + "net" + "unsafe" +) + +// An Option represents a sticky socket option. +type Option struct { + Level int // level + Name int // name; must be equal or greater than 1 + Len int // length of value in bytes; must be equal or greater than 1 +} + +// Get reads a value for the option from the kernel. +// It returns the number of bytes written into b. +func (o *Option) Get(c *Conn, b []byte) (int, error) { + if o.Name < 1 || o.Len < 1 { + return 0, errors.New("invalid option") + } + if len(b) < o.Len { + return 0, errors.New("short buffer") + } + return o.get(c, b) +} + +// GetInt returns an integer value for the option. +// +// The Len field of Option must be either 1 or 4. +func (o *Option) GetInt(c *Conn) (int, error) { + if o.Len != 1 && o.Len != 4 { + return 0, errors.New("invalid option") + } + var b []byte + var bb [4]byte + if o.Len == 1 { + b = bb[:1] + } else { + b = bb[:4] + } + n, err := o.get(c, b) + if err != nil { + return 0, err + } + if n != o.Len { + return 0, errors.New("invalid option length") + } + if o.Len == 1 { + return int(b[0]), nil + } + return int(NativeEndian.Uint32(b[:4])), nil +} + +// Set writes the option and value to the kernel. +func (o *Option) Set(c *Conn, b []byte) error { + if o.Name < 1 || o.Len < 1 { + return errors.New("invalid option") + } + if len(b) < o.Len { + return errors.New("short buffer") + } + return o.set(c, b) +} + +// SetInt writes the option and value to the kernel. +// +// The Len field of Option must be either 1 or 4. +func (o *Option) SetInt(c *Conn, v int) error { + if o.Len != 1 && o.Len != 4 { + return errors.New("invalid option") + } + var b []byte + if o.Len == 1 { + b = []byte{byte(v)} + } else { + var bb [4]byte + NativeEndian.PutUint32(bb[:o.Len], uint32(v)) + b = bb[:4] + } + return o.set(c, b) +} + +func controlHeaderLen() int { + return roundup(sizeofCmsghdr) +} + +func controlMessageLen(dataLen int) int { + return roundup(sizeofCmsghdr) + dataLen +} + +// ControlMessageSpace returns the whole length of control message. +func ControlMessageSpace(dataLen int) int { + return roundup(sizeofCmsghdr) + roundup(dataLen) +} + +// A ControlMessage represents the head message in a stream of control +// messages. +// +// A control message comprises of a header, data and a few padding +// fields to conform to the interface to the kernel. +// +// See RFC 3542 for further information. +type ControlMessage []byte + +// Data returns the data field of the control message at the head on +// m. +func (m ControlMessage) Data(dataLen int) []byte { + l := controlHeaderLen() + if len(m) < l || len(m) < l+dataLen { + return nil + } + return m[l : l+dataLen] +} + +// Next returns the control message at the next on m. +// +// Next works only for standard control messages. +func (m ControlMessage) Next(dataLen int) ControlMessage { + l := ControlMessageSpace(dataLen) + if len(m) < l { + return nil + } + return m[l:] +} + +// MarshalHeader marshals the header fields of the control message at +// the head on m. +func (m ControlMessage) MarshalHeader(lvl, typ, dataLen int) error { + if len(m) < controlHeaderLen() { + return errors.New("short message") + } + h := (*cmsghdr)(unsafe.Pointer(&m[0])) + h.set(controlMessageLen(dataLen), lvl, typ) + return nil +} + +// ParseHeader parses and returns the header fields of the control +// message at the head on m. +func (m ControlMessage) ParseHeader() (lvl, typ, dataLen int, err error) { + l := controlHeaderLen() + if len(m) < l { + return 0, 0, 0, errors.New("short message") + } + h := (*cmsghdr)(unsafe.Pointer(&m[0])) + return h.lvl(), h.typ(), int(uint64(h.len()) - uint64(l)), nil +} + +// Marshal marshals the control message at the head on m, and returns +// the next control message. +func (m ControlMessage) Marshal(lvl, typ int, data []byte) (ControlMessage, error) { + l := len(data) + if len(m) < ControlMessageSpace(l) { + return nil, errors.New("short message") + } + h := (*cmsghdr)(unsafe.Pointer(&m[0])) + h.set(controlMessageLen(l), lvl, typ) + if l > 0 { + copy(m.Data(l), data) + } + return m.Next(l), nil +} + +// Parse parses m as a single or multiple control messages. +// +// Parse works for both standard and compatible messages. +func (m ControlMessage) Parse() ([]ControlMessage, error) { + var ms []ControlMessage + for len(m) >= controlHeaderLen() { + h := (*cmsghdr)(unsafe.Pointer(&m[0])) + l := h.len() + if l <= 0 { + return nil, errors.New("invalid header length") + } + if uint64(l) < uint64(controlHeaderLen()) { + return nil, errors.New("invalid message length") + } + if uint64(l) > uint64(len(m)) { + return nil, errors.New("short buffer") + } + // On message reception: + // + // |<- ControlMessageSpace --------------->| + // |<- controlMessageLen ---------->| | + // |<- controlHeaderLen ->| | | + // +---------------+------+---------+------+ + // | Header | PadH | Data | PadD | + // +---------------+------+---------+------+ + // + // On compatible message reception: + // + // | ... |<- controlMessageLen ----------->| + // | ... |<- controlHeaderLen ->| | + // +-----+---------------+------+----------+ + // | ... | Header | PadH | Data | + // +-----+---------------+------+----------+ + ms = append(ms, ControlMessage(m[:l])) + ll := l - controlHeaderLen() + if len(m) >= ControlMessageSpace(ll) { + m = m[ControlMessageSpace(ll):] + } else { + m = m[controlMessageLen(ll):] + } + } + return ms, nil +} + +// NewControlMessage returns a new stream of control messages. +func NewControlMessage(dataLen []int) ControlMessage { + var l int + for i := range dataLen { + l += ControlMessageSpace(dataLen[i]) + } + return make([]byte, l) +} + +// A Message represents an IO message. +type Message struct { + // When writing, the Buffers field must contain at least one + // byte to write. + // When reading, the Buffers field will always contain a byte + // to read. + Buffers [][]byte + + // OOB contains protocol-specific control or miscellaneous + // ancillary data known as out-of-band data. + OOB []byte + + // Addr specifies a destination address when writing. + // It can be nil when the underlying protocol of the raw + // connection uses connection-oriented communication. + // After a successful read, it may contain the source address + // on the received packet. + Addr net.Addr + + N int // # of bytes read or written from/to Buffers + NN int // # of bytes read or written from/to OOB + Flags int // protocol-specific information on the received message +} + +// RecvMsg wraps recvmsg system call. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_PEEK. +func (c *Conn) RecvMsg(m *Message, flags int) error { + return c.recvMsg(m, flags) +} + +// SendMsg wraps sendmsg system call. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_DONTROUTE. +func (c *Conn) SendMsg(m *Message, flags int) error { + return c.sendMsg(m, flags) +} + +// RecvMsgs wraps recvmmsg system call. +// +// It returns the number of processed messages. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_PEEK. +// +// Only Linux supports this. +func (c *Conn) RecvMsgs(ms []Message, flags int) (int, error) { + return c.recvMsgs(ms, flags) +} + +// SendMsgs wraps sendmmsg system call. +// +// It returns the number of processed messages. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_DONTROUTE. +// +// Only Linux supports this. +func (c *Conn) SendMsgs(ms []Message, flags int) (int, error) { + return c.sendMsgs(ms, flags) +} diff --git a/vendor/golang.org/x/net/internal/socket/sys.go b/vendor/golang.org/x/net/internal/socket/sys.go new file mode 100644 index 00000000..4f0eead1 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys.go @@ -0,0 +1,33 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +import ( + "encoding/binary" + "unsafe" +) + +var ( + // NativeEndian is the machine native endian implementation of + // ByteOrder. + NativeEndian binary.ByteOrder + + kernelAlign int +) + +func init() { + i := uint32(1) + b := (*[4]byte)(unsafe.Pointer(&i)) + if b[0] == 1 { + NativeEndian = binary.LittleEndian + } else { + NativeEndian = binary.BigEndian + } + kernelAlign = probeProtocolStack() +} + +func roundup(l int) int { + return (l + kernelAlign - 1) & ^(kernelAlign - 1) +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_bsd.go b/vendor/golang.org/x/net/internal/socket/sys_bsd.go new file mode 100644 index 00000000..f13e14ff --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_bsd.go @@ -0,0 +1,17 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd openbsd + +package socket + +import "errors" + +func recvmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func sendmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_bsdvar.go b/vendor/golang.org/x/net/internal/socket/sys_bsdvar.go new file mode 100644 index 00000000..f723fa36 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_bsdvar.go @@ -0,0 +1,14 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd netbsd openbsd + +package socket + +import "unsafe" + +func probeProtocolStack() int { + var p uintptr + return int(unsafe.Sizeof(p)) +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_darwin.go b/vendor/golang.org/x/net/internal/socket/sys_darwin.go new file mode 100644 index 00000000..b17d223b --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_darwin.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +func probeProtocolStack() int { return 4 } diff --git a/vendor/golang.org/x/net/internal/socket/sys_dragonfly.go b/vendor/golang.org/x/net/internal/socket/sys_dragonfly.go new file mode 100644 index 00000000..b17d223b --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_dragonfly.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +func probeProtocolStack() int { return 4 } diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux.go b/vendor/golang.org/x/net/internal/socket/sys_linux.go new file mode 100644 index 00000000..1559521e --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux.go @@ -0,0 +1,27 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux,!s390x,!386 + +package socket + +import ( + "syscall" + "unsafe" +) + +func probeProtocolStack() int { + var p uintptr + return int(unsafe.Sizeof(p)) +} + +func recvmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + n, _, errno := syscall.Syscall6(sysRECVMMSG, s, uintptr(unsafe.Pointer(&hs[0])), uintptr(len(hs)), uintptr(flags), 0, 0) + return int(n), errnoErr(errno) +} + +func sendmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + n, _, errno := syscall.Syscall6(sysSENDMMSG, s, uintptr(unsafe.Pointer(&hs[0])), uintptr(len(hs)), uintptr(flags), 0, 0) + return int(n), errnoErr(errno) +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_386.go b/vendor/golang.org/x/net/internal/socket/sys_linux_386.go new file mode 100644 index 00000000..235b2cc0 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_386.go @@ -0,0 +1,55 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +import ( + "syscall" + "unsafe" +) + +func probeProtocolStack() int { return 4 } + +const ( + sysSETSOCKOPT = 0xe + sysGETSOCKOPT = 0xf + sysSENDMSG = 0x10 + sysRECVMSG = 0x11 + sysRECVMMSG = 0x13 + sysSENDMMSG = 0x14 +) + +func socketcall(call, a0, a1, a2, a3, a4, a5 uintptr) (uintptr, syscall.Errno) +func rawsocketcall(call, a0, a1, a2, a3, a4, a5 uintptr) (uintptr, syscall.Errno) + +func getsockopt(s uintptr, level, name int, b []byte) (int, error) { + l := uint32(len(b)) + _, errno := socketcall(sysGETSOCKOPT, s, uintptr(level), uintptr(name), uintptr(unsafe.Pointer(&b[0])), uintptr(unsafe.Pointer(&l)), 0) + return int(l), errnoErr(errno) +} + +func setsockopt(s uintptr, level, name int, b []byte) error { + _, errno := socketcall(sysSETSOCKOPT, s, uintptr(level), uintptr(name), uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), 0) + return errnoErr(errno) +} + +func recvmsg(s uintptr, h *msghdr, flags int) (int, error) { + n, errno := socketcall(sysRECVMSG, s, uintptr(unsafe.Pointer(h)), uintptr(flags), 0, 0, 0) + return int(n), errnoErr(errno) +} + +func sendmsg(s uintptr, h *msghdr, flags int) (int, error) { + n, errno := socketcall(sysSENDMSG, s, uintptr(unsafe.Pointer(h)), uintptr(flags), 0, 0, 0) + return int(n), errnoErr(errno) +} + +func recvmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + n, errno := socketcall(sysRECVMMSG, s, uintptr(unsafe.Pointer(&hs[0])), uintptr(len(hs)), uintptr(flags), 0, 0) + return int(n), errnoErr(errno) +} + +func sendmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + n, errno := socketcall(sysSENDMMSG, s, uintptr(unsafe.Pointer(&hs[0])), uintptr(len(hs)), uintptr(flags), 0, 0) + return int(n), errnoErr(errno) +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_386.s b/vendor/golang.org/x/net/internal/socket/sys_linux_386.s new file mode 100644 index 00000000..93e7d75e --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_386.s @@ -0,0 +1,11 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT ·socketcall(SB),NOSPLIT,$0-36 + JMP syscall·socketcall(SB) + +TEXT ·rawsocketcall(SB),NOSPLIT,$0-36 + JMP syscall·rawsocketcall(SB) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_amd64.go b/vendor/golang.org/x/net/internal/socket/sys_linux_amd64.go new file mode 100644 index 00000000..9decee2e --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_amd64.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0x12b + sysSENDMMSG = 0x133 +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_arm.go b/vendor/golang.org/x/net/internal/socket/sys_linux_arm.go new file mode 100644 index 00000000..d753b436 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_arm.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0x16d + sysSENDMMSG = 0x176 +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_arm64.go b/vendor/golang.org/x/net/internal/socket/sys_linux_arm64.go new file mode 100644 index 00000000..b6708943 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_arm64.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0xf3 + sysSENDMMSG = 0x10d +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_mips.go b/vendor/golang.org/x/net/internal/socket/sys_linux_mips.go new file mode 100644 index 00000000..9c0d7401 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_mips.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0x10ef + sysSENDMMSG = 0x10f7 +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_mips64.go b/vendor/golang.org/x/net/internal/socket/sys_linux_mips64.go new file mode 100644 index 00000000..071a4aba --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_mips64.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0x14ae + sysSENDMMSG = 0x14b6 +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_mips64le.go b/vendor/golang.org/x/net/internal/socket/sys_linux_mips64le.go new file mode 100644 index 00000000..071a4aba --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_mips64le.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0x14ae + sysSENDMMSG = 0x14b6 +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_mipsle.go b/vendor/golang.org/x/net/internal/socket/sys_linux_mipsle.go new file mode 100644 index 00000000..9c0d7401 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_mipsle.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0x10ef + sysSENDMMSG = 0x10f7 +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_ppc64.go b/vendor/golang.org/x/net/internal/socket/sys_linux_ppc64.go new file mode 100644 index 00000000..21c1e3f0 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_ppc64.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0x157 + sysSENDMMSG = 0x15d +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_ppc64le.go b/vendor/golang.org/x/net/internal/socket/sys_linux_ppc64le.go new file mode 100644 index 00000000..21c1e3f0 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_ppc64le.go @@ -0,0 +1,10 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +const ( + sysRECVMMSG = 0x157 + sysSENDMMSG = 0x15d +) diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_s390x.go b/vendor/golang.org/x/net/internal/socket/sys_linux_s390x.go new file mode 100644 index 00000000..327979ef --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_s390x.go @@ -0,0 +1,55 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +import ( + "syscall" + "unsafe" +) + +func probeProtocolStack() int { return 8 } + +const ( + sysSETSOCKOPT = 0xe + sysGETSOCKOPT = 0xf + sysSENDMSG = 0x10 + sysRECVMSG = 0x11 + sysRECVMMSG = 0x13 + sysSENDMMSG = 0x14 +) + +func socketcall(call, a0, a1, a2, a3, a4, a5 uintptr) (uintptr, syscall.Errno) +func rawsocketcall(call, a0, a1, a2, a3, a4, a5 uintptr) (uintptr, syscall.Errno) + +func getsockopt(s uintptr, level, name int, b []byte) (int, error) { + l := uint32(len(b)) + _, errno := socketcall(sysGETSOCKOPT, s, uintptr(level), uintptr(name), uintptr(unsafe.Pointer(&b[0])), uintptr(unsafe.Pointer(&l)), 0) + return int(l), errnoErr(errno) +} + +func setsockopt(s uintptr, level, name int, b []byte) error { + _, errno := socketcall(sysSETSOCKOPT, s, uintptr(level), uintptr(name), uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), 0) + return errnoErr(errno) +} + +func recvmsg(s uintptr, h *msghdr, flags int) (int, error) { + n, errno := socketcall(sysRECVMSG, s, uintptr(unsafe.Pointer(h)), uintptr(flags), 0, 0, 0) + return int(n), errnoErr(errno) +} + +func sendmsg(s uintptr, h *msghdr, flags int) (int, error) { + n, errno := socketcall(sysSENDMSG, s, uintptr(unsafe.Pointer(h)), uintptr(flags), 0, 0, 0) + return int(n), errnoErr(errno) +} + +func recvmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + n, errno := socketcall(sysRECVMMSG, s, uintptr(unsafe.Pointer(&hs[0])), uintptr(len(hs)), uintptr(flags), 0, 0) + return int(n), errnoErr(errno) +} + +func sendmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + n, errno := socketcall(sysSENDMMSG, s, uintptr(unsafe.Pointer(&hs[0])), uintptr(len(hs)), uintptr(flags), 0, 0) + return int(n), errnoErr(errno) +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_linux_s390x.s b/vendor/golang.org/x/net/internal/socket/sys_linux_s390x.s new file mode 100644 index 00000000..06d75628 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_linux_s390x.s @@ -0,0 +1,11 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT ·socketcall(SB),NOSPLIT,$0-72 + JMP syscall·socketcall(SB) + +TEXT ·rawsocketcall(SB),NOSPLIT,$0-72 + JMP syscall·rawsocketcall(SB) diff --git a/vendor/golang.org/x/net/internal/socket/sys_netbsd.go b/vendor/golang.org/x/net/internal/socket/sys_netbsd.go new file mode 100644 index 00000000..431851c1 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_netbsd.go @@ -0,0 +1,25 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +import ( + "syscall" + "unsafe" +) + +const ( + sysRECVMMSG = 0x1db + sysSENDMMSG = 0x1dc +) + +func recvmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + n, _, errno := syscall.Syscall6(sysRECVMMSG, s, uintptr(unsafe.Pointer(&hs[0])), uintptr(len(hs)), uintptr(flags), 0, 0) + return int(n), errnoErr(errno) +} + +func sendmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + n, _, errno := syscall.Syscall6(sysSENDMMSG, s, uintptr(unsafe.Pointer(&hs[0])), uintptr(len(hs)), uintptr(flags), 0, 0) + return int(n), errnoErr(errno) +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_posix.go b/vendor/golang.org/x/net/internal/socket/sys_posix.go new file mode 100644 index 00000000..dc130c27 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_posix.go @@ -0,0 +1,168 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 +// +build darwin dragonfly freebsd linux netbsd openbsd solaris windows + +package socket + +import ( + "encoding/binary" + "errors" + "net" + "runtime" + "strconv" + "sync" + "time" +) + +func marshalInetAddr(a net.Addr) []byte { + switch a := a.(type) { + case *net.TCPAddr: + return marshalSockaddr(a.IP, a.Port, a.Zone) + case *net.UDPAddr: + return marshalSockaddr(a.IP, a.Port, a.Zone) + case *net.IPAddr: + return marshalSockaddr(a.IP, 0, a.Zone) + default: + return nil + } +} + +func marshalSockaddr(ip net.IP, port int, zone string) []byte { + if ip4 := ip.To4(); ip4 != nil { + b := make([]byte, sizeofSockaddrInet) + switch runtime.GOOS { + case "android", "linux", "solaris", "windows": + NativeEndian.PutUint16(b[:2], uint16(sysAF_INET)) + default: + b[0] = sizeofSockaddrInet + b[1] = sysAF_INET + } + binary.BigEndian.PutUint16(b[2:4], uint16(port)) + copy(b[4:8], ip4) + return b + } + if ip6 := ip.To16(); ip6 != nil && ip.To4() == nil { + b := make([]byte, sizeofSockaddrInet6) + switch runtime.GOOS { + case "android", "linux", "solaris", "windows": + NativeEndian.PutUint16(b[:2], uint16(sysAF_INET6)) + default: + b[0] = sizeofSockaddrInet6 + b[1] = sysAF_INET6 + } + binary.BigEndian.PutUint16(b[2:4], uint16(port)) + copy(b[8:24], ip6) + if zone != "" { + NativeEndian.PutUint32(b[24:28], uint32(zoneCache.index(zone))) + } + return b + } + return nil +} + +func parseInetAddr(b []byte, network string) (net.Addr, error) { + if len(b) < 2 { + return nil, errors.New("invalid address") + } + var af int + switch runtime.GOOS { + case "android", "linux", "solaris", "windows": + af = int(NativeEndian.Uint16(b[:2])) + default: + af = int(b[1]) + } + var ip net.IP + var zone string + if af == sysAF_INET { + if len(b) < sizeofSockaddrInet { + return nil, errors.New("short address") + } + ip = make(net.IP, net.IPv4len) + copy(ip, b[4:8]) + } + if af == sysAF_INET6 { + if len(b) < sizeofSockaddrInet6 { + return nil, errors.New("short address") + } + ip = make(net.IP, net.IPv6len) + copy(ip, b[8:24]) + if id := int(NativeEndian.Uint32(b[24:28])); id > 0 { + zone = zoneCache.name(id) + } + } + switch network { + case "tcp", "tcp4", "tcp6": + return &net.TCPAddr{IP: ip, Port: int(binary.BigEndian.Uint16(b[2:4])), Zone: zone}, nil + case "udp", "udp4", "udp6": + return &net.UDPAddr{IP: ip, Port: int(binary.BigEndian.Uint16(b[2:4])), Zone: zone}, nil + default: + return &net.IPAddr{IP: ip, Zone: zone}, nil + } +} + +// An ipv6ZoneCache represents a cache holding partial network +// interface information. It is used for reducing the cost of IPv6 +// addressing scope zone resolution. +// +// Multiple names sharing the index are managed by first-come +// first-served basis for consistency. +type ipv6ZoneCache struct { + sync.RWMutex // guard the following + lastFetched time.Time // last time routing information was fetched + toIndex map[string]int // interface name to its index + toName map[int]string // interface index to its name +} + +var zoneCache = ipv6ZoneCache{ + toIndex: make(map[string]int), + toName: make(map[int]string), +} + +func (zc *ipv6ZoneCache) update(ift []net.Interface) { + zc.Lock() + defer zc.Unlock() + now := time.Now() + if zc.lastFetched.After(now.Add(-60 * time.Second)) { + return + } + zc.lastFetched = now + if len(ift) == 0 { + var err error + if ift, err = net.Interfaces(); err != nil { + return + } + } + zc.toIndex = make(map[string]int, len(ift)) + zc.toName = make(map[int]string, len(ift)) + for _, ifi := range ift { + zc.toIndex[ifi.Name] = ifi.Index + if _, ok := zc.toName[ifi.Index]; !ok { + zc.toName[ifi.Index] = ifi.Name + } + } +} + +func (zc *ipv6ZoneCache) name(zone int) string { + zoneCache.update(nil) + zoneCache.RLock() + defer zoneCache.RUnlock() + name, ok := zoneCache.toName[zone] + if !ok { + name = strconv.Itoa(zone) + } + return name +} + +func (zc *ipv6ZoneCache) index(zone string) int { + zoneCache.update(nil) + zoneCache.RLock() + defer zoneCache.RUnlock() + index, ok := zoneCache.toIndex[zone] + if !ok { + index, _ = strconv.Atoi(zone) + } + return index +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_solaris.go b/vendor/golang.org/x/net/internal/socket/sys_solaris.go new file mode 100644 index 00000000..cced74e6 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_solaris.go @@ -0,0 +1,71 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +import ( + "errors" + "runtime" + "syscall" + "unsafe" +) + +func probeProtocolStack() int { + switch runtime.GOARCH { + case "amd64": + return 4 + default: + var p uintptr + return int(unsafe.Sizeof(p)) + } +} + +//go:cgo_import_dynamic libc___xnet_getsockopt __xnet_getsockopt "libsocket.so" +//go:cgo_import_dynamic libc_setsockopt setsockopt "libsocket.so" +//go:cgo_import_dynamic libc___xnet_recvmsg __xnet_recvmsg "libsocket.so" +//go:cgo_import_dynamic libc___xnet_sendmsg __xnet_sendmsg "libsocket.so" + +//go:linkname procGetsockopt libc___xnet_getsockopt +//go:linkname procSetsockopt libc_setsockopt +//go:linkname procRecvmsg libc___xnet_recvmsg +//go:linkname procSendmsg libc___xnet_sendmsg + +var ( + procGetsockopt uintptr + procSetsockopt uintptr + procRecvmsg uintptr + procSendmsg uintptr +) + +func sysvicall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (uintptr, uintptr, syscall.Errno) +func rawSysvicall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (uintptr, uintptr, syscall.Errno) + +func getsockopt(s uintptr, level, name int, b []byte) (int, error) { + l := uint32(len(b)) + _, _, errno := sysvicall6(uintptr(unsafe.Pointer(&procGetsockopt)), 5, s, uintptr(level), uintptr(name), uintptr(unsafe.Pointer(&b[0])), uintptr(unsafe.Pointer(&l)), 0) + return int(l), errnoErr(errno) +} + +func setsockopt(s uintptr, level, name int, b []byte) error { + _, _, errno := sysvicall6(uintptr(unsafe.Pointer(&procSetsockopt)), 5, s, uintptr(level), uintptr(name), uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), 0) + return errnoErr(errno) +} + +func recvmsg(s uintptr, h *msghdr, flags int) (int, error) { + n, _, errno := sysvicall6(uintptr(unsafe.Pointer(&procRecvmsg)), 3, s, uintptr(unsafe.Pointer(h)), uintptr(flags), 0, 0, 0) + return int(n), errnoErr(errno) +} + +func sendmsg(s uintptr, h *msghdr, flags int) (int, error) { + n, _, errno := sysvicall6(uintptr(unsafe.Pointer(&procSendmsg)), 3, s, uintptr(unsafe.Pointer(h)), uintptr(flags), 0, 0, 0) + return int(n), errnoErr(errno) +} + +func recvmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func sendmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_solaris_amd64.s b/vendor/golang.org/x/net/internal/socket/sys_solaris_amd64.s new file mode 100644 index 00000000..a18ac5ed --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_solaris_amd64.s @@ -0,0 +1,11 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT ·sysvicall6(SB),NOSPLIT,$0-88 + JMP syscall·sysvicall6(SB) + +TEXT ·rawSysvicall6(SB),NOSPLIT,$0-88 + JMP syscall·rawSysvicall6(SB) diff --git a/vendor/golang.org/x/net/internal/socket/sys_stub.go b/vendor/golang.org/x/net/internal/socket/sys_stub.go new file mode 100644 index 00000000..d9f06d00 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_stub.go @@ -0,0 +1,64 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package socket + +import ( + "errors" + "net" + "runtime" + "unsafe" +) + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +func probeProtocolStack() int { + switch runtime.GOARCH { + case "amd64p32", "mips64p32": + return 4 + default: + var p uintptr + return int(unsafe.Sizeof(p)) + } +} + +func marshalInetAddr(ip net.IP, port int, zone string) []byte { + return nil +} + +func parseInetAddr(b []byte, network string) (net.Addr, error) { + return nil, errors.New("not implemented") +} + +func getsockopt(s uintptr, level, name int, b []byte) (int, error) { + return 0, errors.New("not implemented") +} + +func setsockopt(s uintptr, level, name int, b []byte) error { + return errors.New("not implemented") +} + +func recvmsg(s uintptr, h *msghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func sendmsg(s uintptr, h *msghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func recvmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func sendmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_unix.go b/vendor/golang.org/x/net/internal/socket/sys_unix.go new file mode 100644 index 00000000..18eba308 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_unix.go @@ -0,0 +1,33 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux,!s390x,!386 netbsd openbsd + +package socket + +import ( + "syscall" + "unsafe" +) + +func getsockopt(s uintptr, level, name int, b []byte) (int, error) { + l := uint32(len(b)) + _, _, errno := syscall.Syscall6(syscall.SYS_GETSOCKOPT, s, uintptr(level), uintptr(name), uintptr(unsafe.Pointer(&b[0])), uintptr(unsafe.Pointer(&l)), 0) + return int(l), errnoErr(errno) +} + +func setsockopt(s uintptr, level, name int, b []byte) error { + _, _, errno := syscall.Syscall6(syscall.SYS_SETSOCKOPT, s, uintptr(level), uintptr(name), uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), 0) + return errnoErr(errno) +} + +func recvmsg(s uintptr, h *msghdr, flags int) (int, error) { + n, _, errno := syscall.Syscall(syscall.SYS_RECVMSG, s, uintptr(unsafe.Pointer(h)), uintptr(flags)) + return int(n), errnoErr(errno) +} + +func sendmsg(s uintptr, h *msghdr, flags int) (int, error) { + n, _, errno := syscall.Syscall(syscall.SYS_SENDMSG, s, uintptr(unsafe.Pointer(h)), uintptr(flags)) + return int(n), errnoErr(errno) +} diff --git a/vendor/golang.org/x/net/internal/socket/sys_windows.go b/vendor/golang.org/x/net/internal/socket/sys_windows.go new file mode 100644 index 00000000..54a470eb --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/sys_windows.go @@ -0,0 +1,70 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package socket + +import ( + "errors" + "syscall" + "unsafe" +) + +func probeProtocolStack() int { + var p uintptr + return int(unsafe.Sizeof(p)) +} + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x17 + + sysSOCK_RAW = 0x3 +) + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) + +func getsockopt(s uintptr, level, name int, b []byte) (int, error) { + l := uint32(len(b)) + err := syscall.Getsockopt(syscall.Handle(s), int32(level), int32(name), (*byte)(unsafe.Pointer(&b[0])), (*int32)(unsafe.Pointer(&l))) + return int(l), err +} + +func setsockopt(s uintptr, level, name int, b []byte) error { + return syscall.Setsockopt(syscall.Handle(s), int32(level), int32(name), (*byte)(unsafe.Pointer(&b[0])), int32(len(b))) +} + +func recvmsg(s uintptr, h *msghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func sendmsg(s uintptr, h *msghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func recvmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} + +func sendmmsg(s uintptr, hs []mmsghdr, flags int) (int, error) { + return 0, errors.New("not implemented") +} diff --git a/vendor/golang.org/x/net/internal/socket/zsys_darwin_386.go b/vendor/golang.org/x/net/internal/socket/zsys_darwin_386.go new file mode 100644 index 00000000..26f8feff --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_darwin_386.go @@ -0,0 +1,59 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_darwin.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1e + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen int32 + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_darwin_amd64.go b/vendor/golang.org/x/net/internal/socket/zsys_darwin_amd64.go new file mode 100644 index 00000000..e2987f7d --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_darwin_amd64.go @@ -0,0 +1,61 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_darwin.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1e + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen int32 + Pad_cgo_1 [4]byte + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x30 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_darwin_arm.go b/vendor/golang.org/x/net/internal/socket/zsys_darwin_arm.go new file mode 100644 index 00000000..26f8feff --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_darwin_arm.go @@ -0,0 +1,59 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_darwin.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1e + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen int32 + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_darwin_arm64.go b/vendor/golang.org/x/net/internal/socket/zsys_darwin_arm64.go new file mode 100644 index 00000000..e2987f7d --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_darwin_arm64.go @@ -0,0 +1,61 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_darwin.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1e + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen int32 + Pad_cgo_1 [4]byte + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x30 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_dragonfly_amd64.go b/vendor/golang.org/x/net/internal/socket/zsys_dragonfly_amd64.go new file mode 100644 index 00000000..c582abd5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_dragonfly_amd64.go @@ -0,0 +1,61 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_dragonfly.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1c + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen int32 + Pad_cgo_1 [4]byte + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x30 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_freebsd_386.go b/vendor/golang.org/x/net/internal/socket/zsys_freebsd_386.go new file mode 100644 index 00000000..04a24886 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_freebsd_386.go @@ -0,0 +1,59 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1c + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen int32 + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_freebsd_amd64.go b/vendor/golang.org/x/net/internal/socket/zsys_freebsd_amd64.go new file mode 100644 index 00000000..35c7cb9c --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_freebsd_amd64.go @@ -0,0 +1,61 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1c + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen int32 + Pad_cgo_1 [4]byte + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x30 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_freebsd_arm.go b/vendor/golang.org/x/net/internal/socket/zsys_freebsd_arm.go new file mode 100644 index 00000000..04a24886 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_freebsd_arm.go @@ -0,0 +1,59 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1c + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen int32 + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_386.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_386.go new file mode 100644 index 00000000..43020693 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_386.go @@ -0,0 +1,63 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen uint32 + Control *byte + Controllen uint32 + Flags int32 +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofMmsghdr = 0x20 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_amd64.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_amd64.go new file mode 100644 index 00000000..1502f6c5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_amd64.go @@ -0,0 +1,66 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen uint64 + Control *byte + Controllen uint64 + Flags int32 + Pad_cgo_1 [4]byte +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 + Pad_cgo_0 [4]byte +} + +type cmsghdr struct { + Len uint64 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x38 + sizeofMmsghdr = 0x40 + sizeofCmsghdr = 0x10 + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_arm.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_arm.go new file mode 100644 index 00000000..43020693 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_arm.go @@ -0,0 +1,63 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen uint32 + Control *byte + Controllen uint32 + Flags int32 +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofMmsghdr = 0x20 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_arm64.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_arm64.go new file mode 100644 index 00000000..1502f6c5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_arm64.go @@ -0,0 +1,66 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen uint64 + Control *byte + Controllen uint64 + Flags int32 + Pad_cgo_1 [4]byte +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 + Pad_cgo_0 [4]byte +} + +type cmsghdr struct { + Len uint64 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x38 + sizeofMmsghdr = 0x40 + sizeofCmsghdr = 0x10 + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_mips.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_mips.go new file mode 100644 index 00000000..43020693 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_mips.go @@ -0,0 +1,63 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen uint32 + Control *byte + Controllen uint32 + Flags int32 +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofMmsghdr = 0x20 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_mips64.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_mips64.go new file mode 100644 index 00000000..1502f6c5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_mips64.go @@ -0,0 +1,66 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen uint64 + Control *byte + Controllen uint64 + Flags int32 + Pad_cgo_1 [4]byte +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 + Pad_cgo_0 [4]byte +} + +type cmsghdr struct { + Len uint64 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x38 + sizeofMmsghdr = 0x40 + sizeofCmsghdr = 0x10 + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_mips64le.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_mips64le.go new file mode 100644 index 00000000..1502f6c5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_mips64le.go @@ -0,0 +1,66 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen uint64 + Control *byte + Controllen uint64 + Flags int32 + Pad_cgo_1 [4]byte +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 + Pad_cgo_0 [4]byte +} + +type cmsghdr struct { + Len uint64 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x38 + sizeofMmsghdr = 0x40 + sizeofCmsghdr = 0x10 + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_mipsle.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_mipsle.go new file mode 100644 index 00000000..43020693 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_mipsle.go @@ -0,0 +1,63 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen uint32 + Control *byte + Controllen uint32 + Flags int32 +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofMmsghdr = 0x20 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_ppc64.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_ppc64.go new file mode 100644 index 00000000..1502f6c5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_ppc64.go @@ -0,0 +1,66 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen uint64 + Control *byte + Controllen uint64 + Flags int32 + Pad_cgo_1 [4]byte +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 + Pad_cgo_0 [4]byte +} + +type cmsghdr struct { + Len uint64 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x38 + sizeofMmsghdr = 0x40 + sizeofCmsghdr = 0x10 + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_ppc64le.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_ppc64le.go new file mode 100644 index 00000000..1502f6c5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_ppc64le.go @@ -0,0 +1,66 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen uint64 + Control *byte + Controllen uint64 + Flags int32 + Pad_cgo_1 [4]byte +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 + Pad_cgo_0 [4]byte +} + +type cmsghdr struct { + Len uint64 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x38 + sizeofMmsghdr = 0x40 + sizeofCmsghdr = 0x10 + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_linux_s390x.go b/vendor/golang.org/x/net/internal/socket/zsys_linux_s390x.go new file mode 100644 index 00000000..1502f6c5 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_linux_s390x.go @@ -0,0 +1,66 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0xa + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen uint64 + Control *byte + Controllen uint64 + Flags int32 + Pad_cgo_1 [4]byte +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 + Pad_cgo_0 [4]byte +} + +type cmsghdr struct { + Len uint64 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x38 + sizeofMmsghdr = 0x40 + sizeofCmsghdr = 0x10 + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_netbsd_386.go b/vendor/golang.org/x/net/internal/socket/zsys_netbsd_386.go new file mode 100644 index 00000000..db60491f --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_netbsd_386.go @@ -0,0 +1,65 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_netbsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x18 + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen int32 + Control *byte + Controllen uint32 + Flags int32 +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofMmsghdr = 0x20 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_netbsd_amd64.go b/vendor/golang.org/x/net/internal/socket/zsys_netbsd_amd64.go new file mode 100644 index 00000000..2a1a7998 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_netbsd_amd64.go @@ -0,0 +1,68 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_netbsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x18 + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen int32 + Pad_cgo_1 [4]byte + Control *byte + Controllen uint32 + Flags int32 +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 + Pad_cgo_0 [4]byte +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x30 + sizeofMmsghdr = 0x40 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_netbsd_arm.go b/vendor/golang.org/x/net/internal/socket/zsys_netbsd_arm.go new file mode 100644 index 00000000..db60491f --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_netbsd_arm.go @@ -0,0 +1,65 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_netbsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x18 + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen int32 + Control *byte + Controllen uint32 + Flags int32 +} + +type mmsghdr struct { + Hdr msghdr + Len uint32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofMmsghdr = 0x20 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_openbsd_386.go b/vendor/golang.org/x/net/internal/socket/zsys_openbsd_386.go new file mode 100644 index 00000000..1c836361 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_openbsd_386.go @@ -0,0 +1,59 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_openbsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x18 + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen uint32 + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_openbsd_amd64.go b/vendor/golang.org/x/net/internal/socket/zsys_openbsd_amd64.go new file mode 100644 index 00000000..a6c0bf46 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_openbsd_amd64.go @@ -0,0 +1,61 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_openbsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x18 + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen uint32 + Pad_cgo_1 [4]byte + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x30 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_openbsd_arm.go b/vendor/golang.org/x/net/internal/socket/zsys_openbsd_arm.go new file mode 100644 index 00000000..1c836361 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_openbsd_arm.go @@ -0,0 +1,59 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_openbsd.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x18 + + sysSOCK_RAW = 0x3 +) + +type iovec struct { + Base *byte + Len uint32 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Iov *iovec + Iovlen uint32 + Control *byte + Controllen uint32 + Flags int32 +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +const ( + sizeofIovec = 0x8 + sizeofMsghdr = 0x1c + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x1c +) diff --git a/vendor/golang.org/x/net/internal/socket/zsys_solaris_amd64.go b/vendor/golang.org/x/net/internal/socket/zsys_solaris_amd64.go new file mode 100644 index 00000000..327c6329 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/zsys_solaris_amd64.go @@ -0,0 +1,60 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_solaris.go + +package socket + +const ( + sysAF_UNSPEC = 0x0 + sysAF_INET = 0x2 + sysAF_INET6 = 0x1a + + sysSOCK_RAW = 0x4 +) + +type iovec struct { + Base *int8 + Len uint64 +} + +type msghdr struct { + Name *byte + Namelen uint32 + Pad_cgo_0 [4]byte + Iov *iovec + Iovlen int32 + Pad_cgo_1 [4]byte + Accrights *int8 + Accrightslen int32 + Pad_cgo_2 [4]byte +} + +type cmsghdr struct { + Len uint32 + Level int32 + Type int32 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 + X__sin6_src_id uint32 +} + +const ( + sizeofIovec = 0x10 + sizeofMsghdr = 0x30 + sizeofCmsghdr = 0xc + + sizeofSockaddrInet = 0x10 + sizeofSockaddrInet6 = 0x20 +) diff --git a/vendor/golang.org/x/net/ipv4/batch.go b/vendor/golang.org/x/net/ipv4/batch.go new file mode 100644 index 00000000..b4454992 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/batch.go @@ -0,0 +1,191 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +package ipv4 + +import ( + "net" + "runtime" + "syscall" + + "golang.org/x/net/internal/socket" +) + +// BUG(mikio): On Windows, the ReadBatch and WriteBatch methods of +// PacketConn are not implemented. + +// BUG(mikio): On Windows, the ReadBatch and WriteBatch methods of +// RawConn are not implemented. + +// A Message represents an IO message. +// +// type Message struct { +// Buffers [][]byte +// OOB []byte +// Addr net.Addr +// N int +// NN int +// Flags int +// } +// +// The Buffers fields represents a list of contiguous buffers, which +// can be used for vectored IO, for example, putting a header and a +// payload in each slice. +// When writing, the Buffers field must contain at least one byte to +// write. +// When reading, the Buffers field will always contain a byte to read. +// +// The OOB field contains protocol-specific control or miscellaneous +// ancillary data known as out-of-band data. +// It can be nil when not required. +// +// The Addr field specifies a destination address when writing. +// It can be nil when the underlying protocol of the endpoint uses +// connection-oriented communication. +// After a successful read, it may contain the source address on the +// received packet. +// +// The N field indicates the number of bytes read or written from/to +// Buffers. +// +// The NN field indicates the number of bytes read or written from/to +// OOB. +// +// The Flags field contains protocol-specific information on the +// received message. +type Message = socket.Message + +// ReadBatch reads a batch of messages. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_PEEK. +// +// On a successful read it returns the number of messages received, up +// to len(ms). +// +// On Linux, a batch read will be optimized. +// On other platforms, this method will read only a single message. +// +// Unlike the ReadFrom method, it doesn't strip the IPv4 header +// followed by option headers from the received IPv4 datagram when the +// underlying transport is net.IPConn. Each Buffers field of Message +// must be large enough to accommodate an IPv4 header and option +// headers. +func (c *payloadHandler) ReadBatch(ms []Message, flags int) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + switch runtime.GOOS { + case "linux": + n, err := c.RecvMsgs([]socket.Message(ms), flags) + if err != nil { + err = &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + return n, err + default: + n := 1 + err := c.RecvMsg(&ms[0], flags) + if err != nil { + n = 0 + err = &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + return n, err + } +} + +// WriteBatch writes a batch of messages. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_DONTROUTE. +// +// It returns the number of messages written on a successful write. +// +// On Linux, a batch write will be optimized. +// On other platforms, this method will write only a single message. +func (c *payloadHandler) WriteBatch(ms []Message, flags int) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + switch runtime.GOOS { + case "linux": + n, err := c.SendMsgs([]socket.Message(ms), flags) + if err != nil { + err = &net.OpError{Op: "write", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + return n, err + default: + n := 1 + err := c.SendMsg(&ms[0], flags) + if err != nil { + n = 0 + err = &net.OpError{Op: "write", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + return n, err + } +} + +// ReadBatch reads a batch of messages. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_PEEK. +// +// On a successful read it returns the number of messages received, up +// to len(ms). +// +// On Linux, a batch read will be optimized. +// On other platforms, this method will read only a single message. +func (c *packetHandler) ReadBatch(ms []Message, flags int) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + switch runtime.GOOS { + case "linux": + n, err := c.RecvMsgs([]socket.Message(ms), flags) + if err != nil { + err = &net.OpError{Op: "read", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Err: err} + } + return n, err + default: + n := 1 + err := c.RecvMsg(&ms[0], flags) + if err != nil { + n = 0 + err = &net.OpError{Op: "read", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Err: err} + } + return n, err + } +} + +// WriteBatch writes a batch of messages. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_DONTROUTE. +// +// It returns the number of messages written on a successful write. +// +// On Linux, a batch write will be optimized. +// On other platforms, this method will write only a single message. +func (c *packetHandler) WriteBatch(ms []Message, flags int) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + switch runtime.GOOS { + case "linux": + n, err := c.SendMsgs([]socket.Message(ms), flags) + if err != nil { + err = &net.OpError{Op: "write", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Err: err} + } + return n, err + default: + n := 1 + err := c.SendMsg(&ms[0], flags) + if err != nil { + n = 0 + err = &net.OpError{Op: "write", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Err: err} + } + return n, err + } +} diff --git a/vendor/golang.org/x/net/ipv4/control.go b/vendor/golang.org/x/net/ipv4/control.go new file mode 100644 index 00000000..a2b02ca9 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/control.go @@ -0,0 +1,144 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "fmt" + "net" + "sync" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +type rawOpt struct { + sync.RWMutex + cflags ControlFlags +} + +func (c *rawOpt) set(f ControlFlags) { c.cflags |= f } +func (c *rawOpt) clear(f ControlFlags) { c.cflags &^= f } +func (c *rawOpt) isset(f ControlFlags) bool { return c.cflags&f != 0 } + +type ControlFlags uint + +const ( + FlagTTL ControlFlags = 1 << iota // pass the TTL on the received packet + FlagSrc // pass the source address on the received packet + FlagDst // pass the destination address on the received packet + FlagInterface // pass the interface index on the received packet +) + +// A ControlMessage represents per packet basis IP-level socket options. +type ControlMessage struct { + // Receiving socket options: SetControlMessage allows to + // receive the options from the protocol stack using ReadFrom + // method of PacketConn or RawConn. + // + // Specifying socket options: ControlMessage for WriteTo + // method of PacketConn or RawConn allows to send the options + // to the protocol stack. + // + TTL int // time-to-live, receiving only + Src net.IP // source address, specifying only + Dst net.IP // destination address, receiving only + IfIndex int // interface index, must be 1 <= value when specifying +} + +func (cm *ControlMessage) String() string { + if cm == nil { + return "" + } + return fmt.Sprintf("ttl=%d src=%v dst=%v ifindex=%d", cm.TTL, cm.Src, cm.Dst, cm.IfIndex) +} + +// Marshal returns the binary encoding of cm. +func (cm *ControlMessage) Marshal() []byte { + if cm == nil { + return nil + } + var m socket.ControlMessage + if ctlOpts[ctlPacketInfo].name > 0 && (cm.Src.To4() != nil || cm.IfIndex > 0) { + m = socket.NewControlMessage([]int{ctlOpts[ctlPacketInfo].length}) + } + if len(m) > 0 { + ctlOpts[ctlPacketInfo].marshal(m, cm) + } + return m +} + +// Parse parses b as a control message and stores the result in cm. +func (cm *ControlMessage) Parse(b []byte) error { + ms, err := socket.ControlMessage(b).Parse() + if err != nil { + return err + } + for _, m := range ms { + lvl, typ, l, err := m.ParseHeader() + if err != nil { + return err + } + if lvl != iana.ProtocolIP { + continue + } + switch { + case typ == ctlOpts[ctlTTL].name && l >= ctlOpts[ctlTTL].length: + ctlOpts[ctlTTL].parse(cm, m.Data(l)) + case typ == ctlOpts[ctlDst].name && l >= ctlOpts[ctlDst].length: + ctlOpts[ctlDst].parse(cm, m.Data(l)) + case typ == ctlOpts[ctlInterface].name && l >= ctlOpts[ctlInterface].length: + ctlOpts[ctlInterface].parse(cm, m.Data(l)) + case typ == ctlOpts[ctlPacketInfo].name && l >= ctlOpts[ctlPacketInfo].length: + ctlOpts[ctlPacketInfo].parse(cm, m.Data(l)) + } + } + return nil +} + +// NewControlMessage returns a new control message. +// +// The returned message is large enough for options specified by cf. +func NewControlMessage(cf ControlFlags) []byte { + opt := rawOpt{cflags: cf} + var l int + if opt.isset(FlagTTL) && ctlOpts[ctlTTL].name > 0 { + l += socket.ControlMessageSpace(ctlOpts[ctlTTL].length) + } + if ctlOpts[ctlPacketInfo].name > 0 { + if opt.isset(FlagSrc | FlagDst | FlagInterface) { + l += socket.ControlMessageSpace(ctlOpts[ctlPacketInfo].length) + } + } else { + if opt.isset(FlagDst) && ctlOpts[ctlDst].name > 0 { + l += socket.ControlMessageSpace(ctlOpts[ctlDst].length) + } + if opt.isset(FlagInterface) && ctlOpts[ctlInterface].name > 0 { + l += socket.ControlMessageSpace(ctlOpts[ctlInterface].length) + } + } + var b []byte + if l > 0 { + b = make([]byte, l) + } + return b +} + +// Ancillary data socket options +const ( + ctlTTL = iota // header field + ctlSrc // header field + ctlDst // header field + ctlInterface // inbound or outbound interface + ctlPacketInfo // inbound or outbound packet path + ctlMax +) + +// A ctlOpt represents a binding for ancillary data socket option. +type ctlOpt struct { + name int // option name, must be equal or greater than 1 + length int // option length + marshal func([]byte, *ControlMessage) []byte + parse func(*ControlMessage, []byte) +} diff --git a/vendor/golang.org/x/net/ipv4/control_bsd.go b/vendor/golang.org/x/net/ipv4/control_bsd.go new file mode 100644 index 00000000..77e7ad5b --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/control_bsd.go @@ -0,0 +1,40 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd + +package ipv4 + +import ( + "net" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +func marshalDst(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIP, sysIP_RECVDSTADDR, net.IPv4len) + return m.Next(net.IPv4len) +} + +func parseDst(cm *ControlMessage, b []byte) { + if len(cm.Dst) < net.IPv4len { + cm.Dst = make(net.IP, net.IPv4len) + } + copy(cm.Dst, b[:net.IPv4len]) +} + +func marshalInterface(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIP, sysIP_RECVIF, syscall.SizeofSockaddrDatalink) + return m.Next(syscall.SizeofSockaddrDatalink) +} + +func parseInterface(cm *ControlMessage, b []byte) { + sadl := (*syscall.SockaddrDatalink)(unsafe.Pointer(&b[0])) + cm.IfIndex = int(sadl.Index) +} diff --git a/vendor/golang.org/x/net/ipv4/control_pktinfo.go b/vendor/golang.org/x/net/ipv4/control_pktinfo.go new file mode 100644 index 00000000..425338f3 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/control_pktinfo.go @@ -0,0 +1,39 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin linux solaris + +package ipv4 + +import ( + "net" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +func marshalPacketInfo(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIP, sysIP_PKTINFO, sizeofInetPktinfo) + if cm != nil { + pi := (*inetPktinfo)(unsafe.Pointer(&m.Data(sizeofInetPktinfo)[0])) + if ip := cm.Src.To4(); ip != nil { + copy(pi.Spec_dst[:], ip) + } + if cm.IfIndex > 0 { + pi.setIfindex(cm.IfIndex) + } + } + return m.Next(sizeofInetPktinfo) +} + +func parsePacketInfo(cm *ControlMessage, b []byte) { + pi := (*inetPktinfo)(unsafe.Pointer(&b[0])) + cm.IfIndex = int(pi.Ifindex) + if len(cm.Dst) < net.IPv4len { + cm.Dst = make(net.IP, net.IPv4len) + } + copy(cm.Dst, pi.Addr[:]) +} diff --git a/vendor/golang.org/x/net/ipv4/control_stub.go b/vendor/golang.org/x/net/ipv4/control_stub.go new file mode 100644 index 00000000..5a2f7d8d --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/control_stub.go @@ -0,0 +1,13 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package ipv4 + +import "golang.org/x/net/internal/socket" + +func setControlMessage(c *socket.Conn, opt *rawOpt, cf ControlFlags, on bool) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv4/control_unix.go b/vendor/golang.org/x/net/ipv4/control_unix.go new file mode 100644 index 00000000..e1ae8167 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/control_unix.go @@ -0,0 +1,73 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package ipv4 + +import ( + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +func setControlMessage(c *socket.Conn, opt *rawOpt, cf ControlFlags, on bool) error { + opt.Lock() + defer opt.Unlock() + if so, ok := sockOpts[ssoReceiveTTL]; ok && cf&FlagTTL != 0 { + if err := so.SetInt(c, boolint(on)); err != nil { + return err + } + if on { + opt.set(FlagTTL) + } else { + opt.clear(FlagTTL) + } + } + if so, ok := sockOpts[ssoPacketInfo]; ok { + if cf&(FlagSrc|FlagDst|FlagInterface) != 0 { + if err := so.SetInt(c, boolint(on)); err != nil { + return err + } + if on { + opt.set(cf & (FlagSrc | FlagDst | FlagInterface)) + } else { + opt.clear(cf & (FlagSrc | FlagDst | FlagInterface)) + } + } + } else { + if so, ok := sockOpts[ssoReceiveDst]; ok && cf&FlagDst != 0 { + if err := so.SetInt(c, boolint(on)); err != nil { + return err + } + if on { + opt.set(FlagDst) + } else { + opt.clear(FlagDst) + } + } + if so, ok := sockOpts[ssoReceiveInterface]; ok && cf&FlagInterface != 0 { + if err := so.SetInt(c, boolint(on)); err != nil { + return err + } + if on { + opt.set(FlagInterface) + } else { + opt.clear(FlagInterface) + } + } + } + return nil +} + +func marshalTTL(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIP, sysIP_RECVTTL, 1) + return m.Next(1) +} + +func parseTTL(cm *ControlMessage, b []byte) { + cm.TTL = int(*(*byte)(unsafe.Pointer(&b[:1][0]))) +} diff --git a/vendor/golang.org/x/net/ipv4/control_windows.go b/vendor/golang.org/x/net/ipv4/control_windows.go new file mode 100644 index 00000000..ce55c664 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/control_windows.go @@ -0,0 +1,16 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "syscall" + + "golang.org/x/net/internal/socket" +) + +func setControlMessage(c *socket.Conn, opt *rawOpt, cf ControlFlags, on bool) error { + // TODO(mikio): implement this + return syscall.EWINDOWS +} diff --git a/vendor/golang.org/x/net/ipv4/defs_darwin.go b/vendor/golang.org/x/net/ipv4/defs_darwin.go new file mode 100644 index 00000000..c8f2e05b --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_darwin.go @@ -0,0 +1,77 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include + +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_STRIPHDR = C.IP_STRIPHDR + sysIP_RECVTTL = C.IP_RECVTTL + sysIP_BOUND_IF = C.IP_BOUND_IF + sysIP_PKTINFO = C.IP_PKTINFO + sysIP_RECVPKTINFO = C.IP_RECVPKTINFO + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + sysIP_MULTICAST_VIF = C.IP_MULTICAST_VIF + sysIP_MULTICAST_IFINDEX = C.IP_MULTICAST_IFINDEX + sysIP_ADD_SOURCE_MEMBERSHIP = C.IP_ADD_SOURCE_MEMBERSHIP + sysIP_DROP_SOURCE_MEMBERSHIP = C.IP_DROP_SOURCE_MEMBERSHIP + sysIP_BLOCK_SOURCE = C.IP_BLOCK_SOURCE + sysIP_UNBLOCK_SOURCE = C.IP_UNBLOCK_SOURCE + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofInetPktinfo = C.sizeof_struct_in_pktinfo + + sizeofIPMreq = C.sizeof_struct_ip_mreq + sizeofIPMreqn = C.sizeof_struct_ip_mreqn + sizeofIPMreqSource = C.sizeof_struct_ip_mreq_source + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet C.struct_sockaddr_in + +type inetPktinfo C.struct_in_pktinfo + +type ipMreq C.struct_ip_mreq + +type ipMreqn C.struct_ip_mreqn + +type ipMreqSource C.struct_ip_mreq_source + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv4/defs_dragonfly.go b/vendor/golang.org/x/net/ipv4/defs_dragonfly.go new file mode 100644 index 00000000..f30544ea --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_dragonfly.go @@ -0,0 +1,38 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_MULTICAST_VIF = C.IP_MULTICAST_VIF + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + + sizeofIPMreq = C.sizeof_struct_ip_mreq +) + +type ipMreq C.struct_ip_mreq diff --git a/vendor/golang.org/x/net/ipv4/defs_freebsd.go b/vendor/golang.org/x/net/ipv4/defs_freebsd.go new file mode 100644 index 00000000..4dd57d86 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_freebsd.go @@ -0,0 +1,75 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include + +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_SENDSRCADDR = C.IP_SENDSRCADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_ONESBCAST = C.IP_ONESBCAST + sysIP_BINDANY = C.IP_BINDANY + sysIP_RECVTTL = C.IP_RECVTTL + sysIP_MINTTL = C.IP_MINTTL + sysIP_DONTFRAG = C.IP_DONTFRAG + sysIP_RECVTOS = C.IP_RECVTOS + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + sysIP_MULTICAST_VIF = C.IP_MULTICAST_VIF + sysIP_ADD_SOURCE_MEMBERSHIP = C.IP_ADD_SOURCE_MEMBERSHIP + sysIP_DROP_SOURCE_MEMBERSHIP = C.IP_DROP_SOURCE_MEMBERSHIP + sysIP_BLOCK_SOURCE = C.IP_BLOCK_SOURCE + sysIP_UNBLOCK_SOURCE = C.IP_UNBLOCK_SOURCE + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + + sizeofIPMreq = C.sizeof_struct_ip_mreq + sizeofIPMreqn = C.sizeof_struct_ip_mreqn + sizeofIPMreqSource = C.sizeof_struct_ip_mreq_source + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet C.struct_sockaddr_in + +type ipMreq C.struct_ip_mreq + +type ipMreqn C.struct_ip_mreqn + +type ipMreqSource C.struct_ip_mreq_source + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv4/defs_linux.go b/vendor/golang.org/x/net/ipv4/defs_linux.go new file mode 100644 index 00000000..beb11071 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_linux.go @@ -0,0 +1,122 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include + +#include +#include +#include +#include +#include +*/ +import "C" + +const ( + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_ROUTER_ALERT = C.IP_ROUTER_ALERT + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_PKTINFO = C.IP_PKTINFO + sysIP_PKTOPTIONS = C.IP_PKTOPTIONS + sysIP_MTU_DISCOVER = C.IP_MTU_DISCOVER + sysIP_RECVERR = C.IP_RECVERR + sysIP_RECVTTL = C.IP_RECVTTL + sysIP_RECVTOS = C.IP_RECVTOS + sysIP_MTU = C.IP_MTU + sysIP_FREEBIND = C.IP_FREEBIND + sysIP_TRANSPARENT = C.IP_TRANSPARENT + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_ORIGDSTADDR = C.IP_ORIGDSTADDR + sysIP_RECVORIGDSTADDR = C.IP_RECVORIGDSTADDR + sysIP_MINTTL = C.IP_MINTTL + sysIP_NODEFRAG = C.IP_NODEFRAG + sysIP_UNICAST_IF = C.IP_UNICAST_IF + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + sysIP_UNBLOCK_SOURCE = C.IP_UNBLOCK_SOURCE + sysIP_BLOCK_SOURCE = C.IP_BLOCK_SOURCE + sysIP_ADD_SOURCE_MEMBERSHIP = C.IP_ADD_SOURCE_MEMBERSHIP + sysIP_DROP_SOURCE_MEMBERSHIP = C.IP_DROP_SOURCE_MEMBERSHIP + sysIP_MSFILTER = C.IP_MSFILTER + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + sysMCAST_MSFILTER = C.MCAST_MSFILTER + sysIP_MULTICAST_ALL = C.IP_MULTICAST_ALL + + //sysIP_PMTUDISC_DONT = C.IP_PMTUDISC_DONT + //sysIP_PMTUDISC_WANT = C.IP_PMTUDISC_WANT + //sysIP_PMTUDISC_DO = C.IP_PMTUDISC_DO + //sysIP_PMTUDISC_PROBE = C.IP_PMTUDISC_PROBE + //sysIP_PMTUDISC_INTERFACE = C.IP_PMTUDISC_INTERFACE + //sysIP_PMTUDISC_OMIT = C.IP_PMTUDISC_OMIT + + sysICMP_FILTER = C.ICMP_FILTER + + sysSO_EE_ORIGIN_NONE = C.SO_EE_ORIGIN_NONE + sysSO_EE_ORIGIN_LOCAL = C.SO_EE_ORIGIN_LOCAL + sysSO_EE_ORIGIN_ICMP = C.SO_EE_ORIGIN_ICMP + sysSO_EE_ORIGIN_ICMP6 = C.SO_EE_ORIGIN_ICMP6 + sysSO_EE_ORIGIN_TXSTATUS = C.SO_EE_ORIGIN_TXSTATUS + sysSO_EE_ORIGIN_TIMESTAMPING = C.SO_EE_ORIGIN_TIMESTAMPING + + sysSOL_SOCKET = C.SOL_SOCKET + sysSO_ATTACH_FILTER = C.SO_ATTACH_FILTER + + sizeofKernelSockaddrStorage = C.sizeof_struct___kernel_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofInetPktinfo = C.sizeof_struct_in_pktinfo + sizeofSockExtendedErr = C.sizeof_struct_sock_extended_err + + sizeofIPMreq = C.sizeof_struct_ip_mreq + sizeofIPMreqn = C.sizeof_struct_ip_mreqn + sizeofIPMreqSource = C.sizeof_struct_ip_mreq_source + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPFilter = C.sizeof_struct_icmp_filter + + sizeofSockFprog = C.sizeof_struct_sock_fprog +) + +type kernelSockaddrStorage C.struct___kernel_sockaddr_storage + +type sockaddrInet C.struct_sockaddr_in + +type inetPktinfo C.struct_in_pktinfo + +type sockExtendedErr C.struct_sock_extended_err + +type ipMreq C.struct_ip_mreq + +type ipMreqn C.struct_ip_mreqn + +type ipMreqSource C.struct_ip_mreq_source + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req + +type icmpFilter C.struct_icmp_filter + +type sockFProg C.struct_sock_fprog + +type sockFilter C.struct_sock_filter diff --git a/vendor/golang.org/x/net/ipv4/defs_netbsd.go b/vendor/golang.org/x/net/ipv4/defs_netbsd.go new file mode 100644 index 00000000..8f8af1b8 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_netbsd.go @@ -0,0 +1,37 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + + sizeofIPMreq = C.sizeof_struct_ip_mreq +) + +type ipMreq C.struct_ip_mreq diff --git a/vendor/golang.org/x/net/ipv4/defs_openbsd.go b/vendor/golang.org/x/net/ipv4/defs_openbsd.go new file mode 100644 index 00000000..8f8af1b8 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_openbsd.go @@ -0,0 +1,37 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + + sizeofIPMreq = C.sizeof_struct_ip_mreq +) + +type ipMreq C.struct_ip_mreq diff --git a/vendor/golang.org/x/net/ipv4/defs_solaris.go b/vendor/golang.org/x/net/ipv4/defs_solaris.go new file mode 100644 index 00000000..aeb33e9c --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_solaris.go @@ -0,0 +1,84 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include + +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_RECVSLLA = C.IP_RECVSLLA + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + sysIP_BLOCK_SOURCE = C.IP_BLOCK_SOURCE + sysIP_UNBLOCK_SOURCE = C.IP_UNBLOCK_SOURCE + sysIP_ADD_SOURCE_MEMBERSHIP = C.IP_ADD_SOURCE_MEMBERSHIP + sysIP_DROP_SOURCE_MEMBERSHIP = C.IP_DROP_SOURCE_MEMBERSHIP + sysIP_NEXTHOP = C.IP_NEXTHOP + + sysIP_PKTINFO = C.IP_PKTINFO + sysIP_RECVPKTINFO = C.IP_RECVPKTINFO + sysIP_DONTFRAG = C.IP_DONTFRAG + + sysIP_BOUND_IF = C.IP_BOUND_IF + sysIP_UNSPEC_SRC = C.IP_UNSPEC_SRC + sysIP_BROADCAST_TTL = C.IP_BROADCAST_TTL + sysIP_DHCPINIT_IF = C.IP_DHCPINIT_IF + + sysIP_REUSEADDR = C.IP_REUSEADDR + sysIP_DONTROUTE = C.IP_DONTROUTE + sysIP_BROADCAST = C.IP_BROADCAST + + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofInetPktinfo = C.sizeof_struct_in_pktinfo + + sizeofIPMreq = C.sizeof_struct_ip_mreq + sizeofIPMreqSource = C.sizeof_struct_ip_mreq_source + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet C.struct_sockaddr_in + +type inetPktinfo C.struct_in_pktinfo + +type ipMreq C.struct_ip_mreq + +type ipMreqSource C.struct_ip_mreq_source + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv4/dgramopt.go b/vendor/golang.org/x/net/ipv4/dgramopt.go new file mode 100644 index 00000000..54d77d5f --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/dgramopt.go @@ -0,0 +1,265 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + "syscall" + + "golang.org/x/net/bpf" +) + +// MulticastTTL returns the time-to-live field value for outgoing +// multicast packets. +func (c *dgramOpt) MulticastTTL() (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastTTL] + if !ok { + return 0, errOpNoSupport + } + return so.GetInt(c.Conn) +} + +// SetMulticastTTL sets the time-to-live field value for future +// outgoing multicast packets. +func (c *dgramOpt) SetMulticastTTL(ttl int) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastTTL] + if !ok { + return errOpNoSupport + } + return so.SetInt(c.Conn, ttl) +} + +// MulticastInterface returns the default interface for multicast +// packet transmissions. +func (c *dgramOpt) MulticastInterface() (*net.Interface, error) { + if !c.ok() { + return nil, syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastInterface] + if !ok { + return nil, errOpNoSupport + } + return so.getMulticastInterface(c.Conn) +} + +// SetMulticastInterface sets the default interface for future +// multicast packet transmissions. +func (c *dgramOpt) SetMulticastInterface(ifi *net.Interface) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastInterface] + if !ok { + return errOpNoSupport + } + return so.setMulticastInterface(c.Conn, ifi) +} + +// MulticastLoopback reports whether transmitted multicast packets +// should be copied and send back to the originator. +func (c *dgramOpt) MulticastLoopback() (bool, error) { + if !c.ok() { + return false, syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastLoopback] + if !ok { + return false, errOpNoSupport + } + on, err := so.GetInt(c.Conn) + if err != nil { + return false, err + } + return on == 1, nil +} + +// SetMulticastLoopback sets whether transmitted multicast packets +// should be copied and send back to the originator. +func (c *dgramOpt) SetMulticastLoopback(on bool) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastLoopback] + if !ok { + return errOpNoSupport + } + return so.SetInt(c.Conn, boolint(on)) +} + +// JoinGroup joins the group address group on the interface ifi. +// By default all sources that can cast data to group are accepted. +// It's possible to mute and unmute data transmission from a specific +// source by using ExcludeSourceSpecificGroup and +// IncludeSourceSpecificGroup. +// JoinGroup uses the system assigned multicast interface when ifi is +// nil, although this is not recommended because the assignment +// depends on platforms and sometimes it might require routing +// configuration. +func (c *dgramOpt) JoinGroup(ifi *net.Interface, group net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoJoinGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP4(group) + if grp == nil { + return errMissingAddress + } + return so.setGroup(c.Conn, ifi, grp) +} + +// LeaveGroup leaves the group address group on the interface ifi +// regardless of whether the group is any-source group or +// source-specific group. +func (c *dgramOpt) LeaveGroup(ifi *net.Interface, group net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoLeaveGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP4(group) + if grp == nil { + return errMissingAddress + } + return so.setGroup(c.Conn, ifi, grp) +} + +// JoinSourceSpecificGroup joins the source-specific group comprising +// group and source on the interface ifi. +// JoinSourceSpecificGroup uses the system assigned multicast +// interface when ifi is nil, although this is not recommended because +// the assignment depends on platforms and sometimes it might require +// routing configuration. +func (c *dgramOpt) JoinSourceSpecificGroup(ifi *net.Interface, group, source net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoJoinSourceGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP4(group) + if grp == nil { + return errMissingAddress + } + src := netAddrToIP4(source) + if src == nil { + return errMissingAddress + } + return so.setSourceGroup(c.Conn, ifi, grp, src) +} + +// LeaveSourceSpecificGroup leaves the source-specific group on the +// interface ifi. +func (c *dgramOpt) LeaveSourceSpecificGroup(ifi *net.Interface, group, source net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoLeaveSourceGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP4(group) + if grp == nil { + return errMissingAddress + } + src := netAddrToIP4(source) + if src == nil { + return errMissingAddress + } + return so.setSourceGroup(c.Conn, ifi, grp, src) +} + +// ExcludeSourceSpecificGroup excludes the source-specific group from +// the already joined any-source groups by JoinGroup on the interface +// ifi. +func (c *dgramOpt) ExcludeSourceSpecificGroup(ifi *net.Interface, group, source net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoBlockSourceGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP4(group) + if grp == nil { + return errMissingAddress + } + src := netAddrToIP4(source) + if src == nil { + return errMissingAddress + } + return so.setSourceGroup(c.Conn, ifi, grp, src) +} + +// IncludeSourceSpecificGroup includes the excluded source-specific +// group by ExcludeSourceSpecificGroup again on the interface ifi. +func (c *dgramOpt) IncludeSourceSpecificGroup(ifi *net.Interface, group, source net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoUnblockSourceGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP4(group) + if grp == nil { + return errMissingAddress + } + src := netAddrToIP4(source) + if src == nil { + return errMissingAddress + } + return so.setSourceGroup(c.Conn, ifi, grp, src) +} + +// ICMPFilter returns an ICMP filter. +// Currently only Linux supports this. +func (c *dgramOpt) ICMPFilter() (*ICMPFilter, error) { + if !c.ok() { + return nil, syscall.EINVAL + } + so, ok := sockOpts[ssoICMPFilter] + if !ok { + return nil, errOpNoSupport + } + return so.getICMPFilter(c.Conn) +} + +// SetICMPFilter deploys the ICMP filter. +// Currently only Linux supports this. +func (c *dgramOpt) SetICMPFilter(f *ICMPFilter) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoICMPFilter] + if !ok { + return errOpNoSupport + } + return so.setICMPFilter(c.Conn, f) +} + +// SetBPF attaches a BPF program to the connection. +// +// Only supported on Linux. +func (c *dgramOpt) SetBPF(filter []bpf.RawInstruction) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoAttachFilter] + if !ok { + return errOpNoSupport + } + return so.setBPF(c.Conn, filter) +} diff --git a/vendor/golang.org/x/net/ipv4/doc.go b/vendor/golang.org/x/net/ipv4/doc.go new file mode 100644 index 00000000..b43935a5 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/doc.go @@ -0,0 +1,244 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ipv4 implements IP-level socket options for the Internet +// Protocol version 4. +// +// The package provides IP-level socket options that allow +// manipulation of IPv4 facilities. +// +// The IPv4 protocol and basic host requirements for IPv4 are defined +// in RFC 791 and RFC 1122. +// Host extensions for multicasting and socket interface extensions +// for multicast source filters are defined in RFC 1112 and RFC 3678. +// IGMPv1, IGMPv2 and IGMPv3 are defined in RFC 1112, RFC 2236 and RFC +// 3376. +// Source-specific multicast is defined in RFC 4607. +// +// +// Unicasting +// +// The options for unicasting are available for net.TCPConn, +// net.UDPConn and net.IPConn which are created as network connections +// that use the IPv4 transport. When a single TCP connection carrying +// a data flow of multiple packets needs to indicate the flow is +// important, Conn is used to set the type-of-service field on the +// IPv4 header for each packet. +// +// ln, err := net.Listen("tcp4", "0.0.0.0:1024") +// if err != nil { +// // error handling +// } +// defer ln.Close() +// for { +// c, err := ln.Accept() +// if err != nil { +// // error handling +// } +// go func(c net.Conn) { +// defer c.Close() +// +// The outgoing packets will be labeled DiffServ assured forwarding +// class 1 low drop precedence, known as AF11 packets. +// +// if err := ipv4.NewConn(c).SetTOS(0x28); err != nil { +// // error handling +// } +// if _, err := c.Write(data); err != nil { +// // error handling +// } +// }(c) +// } +// +// +// Multicasting +// +// The options for multicasting are available for net.UDPConn and +// net.IPconn which are created as network connections that use the +// IPv4 transport. A few network facilities must be prepared before +// you begin multicasting, at a minimum joining network interfaces and +// multicast groups. +// +// en0, err := net.InterfaceByName("en0") +// if err != nil { +// // error handling +// } +// en1, err := net.InterfaceByIndex(911) +// if err != nil { +// // error handling +// } +// group := net.IPv4(224, 0, 0, 250) +// +// First, an application listens to an appropriate address with an +// appropriate service port. +// +// c, err := net.ListenPacket("udp4", "0.0.0.0:1024") +// if err != nil { +// // error handling +// } +// defer c.Close() +// +// Second, the application joins multicast groups, starts listening to +// the groups on the specified network interfaces. Note that the +// service port for transport layer protocol does not matter with this +// operation as joining groups affects only network and link layer +// protocols, such as IPv4 and Ethernet. +// +// p := ipv4.NewPacketConn(c) +// if err := p.JoinGroup(en0, &net.UDPAddr{IP: group}); err != nil { +// // error handling +// } +// if err := p.JoinGroup(en1, &net.UDPAddr{IP: group}); err != nil { +// // error handling +// } +// +// The application might set per packet control message transmissions +// between the protocol stack within the kernel. When the application +// needs a destination address on an incoming packet, +// SetControlMessage of PacketConn is used to enable control message +// transmissions. +// +// if err := p.SetControlMessage(ipv4.FlagDst, true); err != nil { +// // error handling +// } +// +// The application could identify whether the received packets are +// of interest by using the control message that contains the +// destination address of the received packet. +// +// b := make([]byte, 1500) +// for { +// n, cm, src, err := p.ReadFrom(b) +// if err != nil { +// // error handling +// } +// if cm.Dst.IsMulticast() { +// if cm.Dst.Equal(group) { +// // joined group, do something +// } else { +// // unknown group, discard +// continue +// } +// } +// +// The application can also send both unicast and multicast packets. +// +// p.SetTOS(0x0) +// p.SetTTL(16) +// if _, err := p.WriteTo(data, nil, src); err != nil { +// // error handling +// } +// dst := &net.UDPAddr{IP: group, Port: 1024} +// for _, ifi := range []*net.Interface{en0, en1} { +// if err := p.SetMulticastInterface(ifi); err != nil { +// // error handling +// } +// p.SetMulticastTTL(2) +// if _, err := p.WriteTo(data, nil, dst); err != nil { +// // error handling +// } +// } +// } +// +// +// More multicasting +// +// An application that uses PacketConn or RawConn may join multiple +// multicast groups. For example, a UDP listener with port 1024 might +// join two different groups across over two different network +// interfaces by using: +// +// c, err := net.ListenPacket("udp4", "0.0.0.0:1024") +// if err != nil { +// // error handling +// } +// defer c.Close() +// p := ipv4.NewPacketConn(c) +// if err := p.JoinGroup(en0, &net.UDPAddr{IP: net.IPv4(224, 0, 0, 248)}); err != nil { +// // error handling +// } +// if err := p.JoinGroup(en0, &net.UDPAddr{IP: net.IPv4(224, 0, 0, 249)}); err != nil { +// // error handling +// } +// if err := p.JoinGroup(en1, &net.UDPAddr{IP: net.IPv4(224, 0, 0, 249)}); err != nil { +// // error handling +// } +// +// It is possible for multiple UDP listeners that listen on the same +// UDP port to join the same multicast group. The net package will +// provide a socket that listens to a wildcard address with reusable +// UDP port when an appropriate multicast address prefix is passed to +// the net.ListenPacket or net.ListenUDP. +// +// c1, err := net.ListenPacket("udp4", "224.0.0.0:1024") +// if err != nil { +// // error handling +// } +// defer c1.Close() +// c2, err := net.ListenPacket("udp4", "224.0.0.0:1024") +// if err != nil { +// // error handling +// } +// defer c2.Close() +// p1 := ipv4.NewPacketConn(c1) +// if err := p1.JoinGroup(en0, &net.UDPAddr{IP: net.IPv4(224, 0, 0, 248)}); err != nil { +// // error handling +// } +// p2 := ipv4.NewPacketConn(c2) +// if err := p2.JoinGroup(en0, &net.UDPAddr{IP: net.IPv4(224, 0, 0, 248)}); err != nil { +// // error handling +// } +// +// Also it is possible for the application to leave or rejoin a +// multicast group on the network interface. +// +// if err := p.LeaveGroup(en0, &net.UDPAddr{IP: net.IPv4(224, 0, 0, 248)}); err != nil { +// // error handling +// } +// if err := p.JoinGroup(en0, &net.UDPAddr{IP: net.IPv4(224, 0, 0, 250)}); err != nil { +// // error handling +// } +// +// +// Source-specific multicasting +// +// An application that uses PacketConn or RawConn on IGMPv3 supported +// platform is able to join source-specific multicast groups. +// The application may use JoinSourceSpecificGroup and +// LeaveSourceSpecificGroup for the operation known as "include" mode, +// +// ssmgroup := net.UDPAddr{IP: net.IPv4(232, 7, 8, 9)} +// ssmsource := net.UDPAddr{IP: net.IPv4(192, 168, 0, 1)}) +// if err := p.JoinSourceSpecificGroup(en0, &ssmgroup, &ssmsource); err != nil { +// // error handling +// } +// if err := p.LeaveSourceSpecificGroup(en0, &ssmgroup, &ssmsource); err != nil { +// // error handling +// } +// +// or JoinGroup, ExcludeSourceSpecificGroup, +// IncludeSourceSpecificGroup and LeaveGroup for the operation known +// as "exclude" mode. +// +// exclsource := net.UDPAddr{IP: net.IPv4(192, 168, 0, 254)} +// if err := p.JoinGroup(en0, &ssmgroup); err != nil { +// // error handling +// } +// if err := p.ExcludeSourceSpecificGroup(en0, &ssmgroup, &exclsource); err != nil { +// // error handling +// } +// if err := p.LeaveGroup(en0, &ssmgroup); err != nil { +// // error handling +// } +// +// Note that it depends on each platform implementation what happens +// when an application which runs on IGMPv3 unsupported platform uses +// JoinSourceSpecificGroup and LeaveSourceSpecificGroup. +// In general the platform tries to fall back to conversations using +// IGMPv1 or IGMPv2 and starts to listen to multicast traffic. +// In the fallback case, ExcludeSourceSpecificGroup and +// IncludeSourceSpecificGroup may return an error. +package ipv4 // import "golang.org/x/net/ipv4" + +// BUG(mikio): This package is not implemented on NaCl and Plan 9. diff --git a/vendor/golang.org/x/net/ipv4/endpoint.go b/vendor/golang.org/x/net/ipv4/endpoint.go new file mode 100644 index 00000000..2ab87736 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/endpoint.go @@ -0,0 +1,187 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + "syscall" + "time" + + "golang.org/x/net/internal/socket" +) + +// BUG(mikio): On Windows, the JoinSourceSpecificGroup, +// LeaveSourceSpecificGroup, ExcludeSourceSpecificGroup and +// IncludeSourceSpecificGroup methods of PacketConn and RawConn are +// not implemented. + +// A Conn represents a network endpoint that uses the IPv4 transport. +// It is used to control basic IP-level socket options such as TOS and +// TTL. +type Conn struct { + genericOpt +} + +type genericOpt struct { + *socket.Conn +} + +func (c *genericOpt) ok() bool { return c != nil && c.Conn != nil } + +// NewConn returns a new Conn. +func NewConn(c net.Conn) *Conn { + cc, _ := socket.NewConn(c) + return &Conn{ + genericOpt: genericOpt{Conn: cc}, + } +} + +// A PacketConn represents a packet network endpoint that uses the +// IPv4 transport. It is used to control several IP-level socket +// options including multicasting. It also provides datagram based +// network I/O methods specific to the IPv4 and higher layer protocols +// such as UDP. +type PacketConn struct { + genericOpt + dgramOpt + payloadHandler +} + +type dgramOpt struct { + *socket.Conn +} + +func (c *dgramOpt) ok() bool { return c != nil && c.Conn != nil } + +// SetControlMessage sets the per packet IP-level socket options. +func (c *PacketConn) SetControlMessage(cf ControlFlags, on bool) error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return setControlMessage(c.dgramOpt.Conn, &c.payloadHandler.rawOpt, cf, on) +} + +// SetDeadline sets the read and write deadlines associated with the +// endpoint. +func (c *PacketConn) SetDeadline(t time.Time) error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return c.payloadHandler.PacketConn.SetDeadline(t) +} + +// SetReadDeadline sets the read deadline associated with the +// endpoint. +func (c *PacketConn) SetReadDeadline(t time.Time) error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return c.payloadHandler.PacketConn.SetReadDeadline(t) +} + +// SetWriteDeadline sets the write deadline associated with the +// endpoint. +func (c *PacketConn) SetWriteDeadline(t time.Time) error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return c.payloadHandler.PacketConn.SetWriteDeadline(t) +} + +// Close closes the endpoint. +func (c *PacketConn) Close() error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return c.payloadHandler.PacketConn.Close() +} + +// NewPacketConn returns a new PacketConn using c as its underlying +// transport. +func NewPacketConn(c net.PacketConn) *PacketConn { + cc, _ := socket.NewConn(c.(net.Conn)) + p := &PacketConn{ + genericOpt: genericOpt{Conn: cc}, + dgramOpt: dgramOpt{Conn: cc}, + payloadHandler: payloadHandler{PacketConn: c, Conn: cc}, + } + return p +} + +// A RawConn represents a packet network endpoint that uses the IPv4 +// transport. It is used to control several IP-level socket options +// including IPv4 header manipulation. It also provides datagram +// based network I/O methods specific to the IPv4 and higher layer +// protocols that handle IPv4 datagram directly such as OSPF, GRE. +type RawConn struct { + genericOpt + dgramOpt + packetHandler +} + +// SetControlMessage sets the per packet IP-level socket options. +func (c *RawConn) SetControlMessage(cf ControlFlags, on bool) error { + if !c.packetHandler.ok() { + return syscall.EINVAL + } + return setControlMessage(c.dgramOpt.Conn, &c.packetHandler.rawOpt, cf, on) +} + +// SetDeadline sets the read and write deadlines associated with the +// endpoint. +func (c *RawConn) SetDeadline(t time.Time) error { + if !c.packetHandler.ok() { + return syscall.EINVAL + } + return c.packetHandler.IPConn.SetDeadline(t) +} + +// SetReadDeadline sets the read deadline associated with the +// endpoint. +func (c *RawConn) SetReadDeadline(t time.Time) error { + if !c.packetHandler.ok() { + return syscall.EINVAL + } + return c.packetHandler.IPConn.SetReadDeadline(t) +} + +// SetWriteDeadline sets the write deadline associated with the +// endpoint. +func (c *RawConn) SetWriteDeadline(t time.Time) error { + if !c.packetHandler.ok() { + return syscall.EINVAL + } + return c.packetHandler.IPConn.SetWriteDeadline(t) +} + +// Close closes the endpoint. +func (c *RawConn) Close() error { + if !c.packetHandler.ok() { + return syscall.EINVAL + } + return c.packetHandler.IPConn.Close() +} + +// NewRawConn returns a new RawConn using c as its underlying +// transport. +func NewRawConn(c net.PacketConn) (*RawConn, error) { + cc, err := socket.NewConn(c.(net.Conn)) + if err != nil { + return nil, err + } + r := &RawConn{ + genericOpt: genericOpt{Conn: cc}, + dgramOpt: dgramOpt{Conn: cc}, + packetHandler: packetHandler{IPConn: c.(*net.IPConn), Conn: cc}, + } + so, ok := sockOpts[ssoHeaderPrepend] + if !ok { + return nil, errOpNoSupport + } + if err := so.SetInt(r.dgramOpt.Conn, boolint(true)); err != nil { + return nil, err + } + return r, nil +} diff --git a/vendor/golang.org/x/net/ipv4/gen.go b/vendor/golang.org/x/net/ipv4/gen.go new file mode 100644 index 00000000..1bb1737f --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/gen.go @@ -0,0 +1,199 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +//go:generate go run gen.go + +// This program generates system adaptation constants and types, +// internet protocol constants and tables by reading template files +// and IANA protocol registries. +package main + +import ( + "bytes" + "encoding/xml" + "fmt" + "go/format" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "runtime" + "strconv" + "strings" +) + +func main() { + if err := genzsys(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := geniana(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func genzsys() error { + defs := "defs_" + runtime.GOOS + ".go" + f, err := os.Open(defs) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + f.Close() + cmd := exec.Command("go", "tool", "cgo", "-godefs", defs) + b, err := cmd.Output() + if err != nil { + return err + } + b, err = format.Source(b) + if err != nil { + return err + } + zsys := "zsys_" + runtime.GOOS + ".go" + switch runtime.GOOS { + case "freebsd", "linux": + zsys = "zsys_" + runtime.GOOS + "_" + runtime.GOARCH + ".go" + } + if err := ioutil.WriteFile(zsys, b, 0644); err != nil { + return err + } + return nil +} + +var registries = []struct { + url string + parse func(io.Writer, io.Reader) error +}{ + { + "https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xml", + parseICMPv4Parameters, + }, +} + +func geniana() error { + var bb bytes.Buffer + fmt.Fprintf(&bb, "// go generate gen.go\n") + fmt.Fprintf(&bb, "// Code generated by the command above; DO NOT EDIT.\n\n") + fmt.Fprintf(&bb, "package ipv4\n\n") + for _, r := range registries { + resp, err := http.Get(r.url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("got HTTP status code %v for %v\n", resp.StatusCode, r.url) + } + if err := r.parse(&bb, resp.Body); err != nil { + return err + } + fmt.Fprintf(&bb, "\n") + } + b, err := format.Source(bb.Bytes()) + if err != nil { + return err + } + if err := ioutil.WriteFile("iana.go", b, 0644); err != nil { + return err + } + return nil +} + +func parseICMPv4Parameters(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var icp icmpv4Parameters + if err := dec.Decode(&icp); err != nil { + return err + } + prs := icp.escape() + fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated) + fmt.Fprintf(w, "const (\n") + for _, pr := range prs { + if pr.Descr == "" { + continue + } + fmt.Fprintf(w, "ICMPType%s ICMPType = %d", pr.Descr, pr.Value) + fmt.Fprintf(w, "// %s\n", pr.OrigDescr) + } + fmt.Fprintf(w, ")\n\n") + fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated) + fmt.Fprintf(w, "var icmpTypes = map[ICMPType]string{\n") + for _, pr := range prs { + if pr.Descr == "" { + continue + } + fmt.Fprintf(w, "%d: %q,\n", pr.Value, strings.ToLower(pr.OrigDescr)) + } + fmt.Fprintf(w, "}\n") + return nil +} + +type icmpv4Parameters struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + Registries []struct { + Title string `xml:"title"` + Records []struct { + Value string `xml:"value"` + Descr string `xml:"description"` + } `xml:"record"` + } `xml:"registry"` +} + +type canonICMPv4ParamRecord struct { + OrigDescr string + Descr string + Value int +} + +func (icp *icmpv4Parameters) escape() []canonICMPv4ParamRecord { + id := -1 + for i, r := range icp.Registries { + if strings.Contains(r.Title, "Type") || strings.Contains(r.Title, "type") { + id = i + break + } + } + if id < 0 { + return nil + } + prs := make([]canonICMPv4ParamRecord, len(icp.Registries[id].Records)) + sr := strings.NewReplacer( + "Messages", "", + "Message", "", + "ICMP", "", + "+", "P", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, pr := range icp.Registries[id].Records { + if strings.Contains(pr.Descr, "Reserved") || + strings.Contains(pr.Descr, "Unassigned") || + strings.Contains(pr.Descr, "Deprecated") || + strings.Contains(pr.Descr, "Experiment") || + strings.Contains(pr.Descr, "experiment") { + continue + } + ss := strings.Split(pr.Descr, "\n") + if len(ss) > 1 { + prs[i].Descr = strings.Join(ss, " ") + } else { + prs[i].Descr = ss[0] + } + s := strings.TrimSpace(prs[i].Descr) + prs[i].OrigDescr = s + prs[i].Descr = sr.Replace(s) + prs[i].Value, _ = strconv.Atoi(pr.Value) + } + return prs +} diff --git a/vendor/golang.org/x/net/ipv4/genericopt.go b/vendor/golang.org/x/net/ipv4/genericopt.go new file mode 100644 index 00000000..119bf841 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/genericopt.go @@ -0,0 +1,57 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import "syscall" + +// TOS returns the type-of-service field value for outgoing packets. +func (c *genericOpt) TOS() (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + so, ok := sockOpts[ssoTOS] + if !ok { + return 0, errOpNoSupport + } + return so.GetInt(c.Conn) +} + +// SetTOS sets the type-of-service field value for future outgoing +// packets. +func (c *genericOpt) SetTOS(tos int) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoTOS] + if !ok { + return errOpNoSupport + } + return so.SetInt(c.Conn, tos) +} + +// TTL returns the time-to-live field value for outgoing packets. +func (c *genericOpt) TTL() (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + so, ok := sockOpts[ssoTTL] + if !ok { + return 0, errOpNoSupport + } + return so.GetInt(c.Conn) +} + +// SetTTL sets the time-to-live field value for future outgoing +// packets. +func (c *genericOpt) SetTTL(ttl int) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoTTL] + if !ok { + return errOpNoSupport + } + return so.SetInt(c.Conn, ttl) +} diff --git a/vendor/golang.org/x/net/ipv4/header.go b/vendor/golang.org/x/net/ipv4/header.go new file mode 100644 index 00000000..8bb0f0f4 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/header.go @@ -0,0 +1,159 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "encoding/binary" + "fmt" + "net" + "runtime" + "syscall" + + "golang.org/x/net/internal/socket" +) + +const ( + Version = 4 // protocol version + HeaderLen = 20 // header length without extension headers + maxHeaderLen = 60 // sensible default, revisit if later RFCs define new usage of version and header length fields +) + +type HeaderFlags int + +const ( + MoreFragments HeaderFlags = 1 << iota // more fragments flag + DontFragment // don't fragment flag +) + +// A Header represents an IPv4 header. +type Header struct { + Version int // protocol version + Len int // header length + TOS int // type-of-service + TotalLen int // packet total length + ID int // identification + Flags HeaderFlags // flags + FragOff int // fragment offset + TTL int // time-to-live + Protocol int // next protocol + Checksum int // checksum + Src net.IP // source address + Dst net.IP // destination address + Options []byte // options, extension headers +} + +func (h *Header) String() string { + if h == nil { + return "" + } + return fmt.Sprintf("ver=%d hdrlen=%d tos=%#x totallen=%d id=%#x flags=%#x fragoff=%#x ttl=%d proto=%d cksum=%#x src=%v dst=%v", h.Version, h.Len, h.TOS, h.TotalLen, h.ID, h.Flags, h.FragOff, h.TTL, h.Protocol, h.Checksum, h.Src, h.Dst) +} + +// Marshal returns the binary encoding of h. +func (h *Header) Marshal() ([]byte, error) { + if h == nil { + return nil, syscall.EINVAL + } + if h.Len < HeaderLen { + return nil, errHeaderTooShort + } + hdrlen := HeaderLen + len(h.Options) + b := make([]byte, hdrlen) + b[0] = byte(Version<<4 | (hdrlen >> 2 & 0x0f)) + b[1] = byte(h.TOS) + flagsAndFragOff := (h.FragOff & 0x1fff) | int(h.Flags<<13) + switch runtime.GOOS { + case "darwin", "dragonfly", "netbsd": + socket.NativeEndian.PutUint16(b[2:4], uint16(h.TotalLen)) + socket.NativeEndian.PutUint16(b[6:8], uint16(flagsAndFragOff)) + case "freebsd": + if freebsdVersion < 1100000 { + socket.NativeEndian.PutUint16(b[2:4], uint16(h.TotalLen)) + socket.NativeEndian.PutUint16(b[6:8], uint16(flagsAndFragOff)) + } else { + binary.BigEndian.PutUint16(b[2:4], uint16(h.TotalLen)) + binary.BigEndian.PutUint16(b[6:8], uint16(flagsAndFragOff)) + } + default: + binary.BigEndian.PutUint16(b[2:4], uint16(h.TotalLen)) + binary.BigEndian.PutUint16(b[6:8], uint16(flagsAndFragOff)) + } + binary.BigEndian.PutUint16(b[4:6], uint16(h.ID)) + b[8] = byte(h.TTL) + b[9] = byte(h.Protocol) + binary.BigEndian.PutUint16(b[10:12], uint16(h.Checksum)) + if ip := h.Src.To4(); ip != nil { + copy(b[12:16], ip[:net.IPv4len]) + } + if ip := h.Dst.To4(); ip != nil { + copy(b[16:20], ip[:net.IPv4len]) + } else { + return nil, errMissingAddress + } + if len(h.Options) > 0 { + copy(b[HeaderLen:], h.Options) + } + return b, nil +} + +// Parse parses b as an IPv4 header and sotres the result in h. +func (h *Header) Parse(b []byte) error { + if h == nil || len(b) < HeaderLen { + return errHeaderTooShort + } + hdrlen := int(b[0]&0x0f) << 2 + if hdrlen > len(b) { + return errBufferTooShort + } + h.Version = int(b[0] >> 4) + h.Len = hdrlen + h.TOS = int(b[1]) + h.ID = int(binary.BigEndian.Uint16(b[4:6])) + h.TTL = int(b[8]) + h.Protocol = int(b[9]) + h.Checksum = int(binary.BigEndian.Uint16(b[10:12])) + h.Src = net.IPv4(b[12], b[13], b[14], b[15]) + h.Dst = net.IPv4(b[16], b[17], b[18], b[19]) + switch runtime.GOOS { + case "darwin", "dragonfly", "netbsd": + h.TotalLen = int(socket.NativeEndian.Uint16(b[2:4])) + hdrlen + h.FragOff = int(socket.NativeEndian.Uint16(b[6:8])) + case "freebsd": + if freebsdVersion < 1100000 { + h.TotalLen = int(socket.NativeEndian.Uint16(b[2:4])) + if freebsdVersion < 1000000 { + h.TotalLen += hdrlen + } + h.FragOff = int(socket.NativeEndian.Uint16(b[6:8])) + } else { + h.TotalLen = int(binary.BigEndian.Uint16(b[2:4])) + h.FragOff = int(binary.BigEndian.Uint16(b[6:8])) + } + default: + h.TotalLen = int(binary.BigEndian.Uint16(b[2:4])) + h.FragOff = int(binary.BigEndian.Uint16(b[6:8])) + } + h.Flags = HeaderFlags(h.FragOff&0xe000) >> 13 + h.FragOff = h.FragOff & 0x1fff + optlen := hdrlen - HeaderLen + if optlen > 0 && len(b) >= hdrlen { + if cap(h.Options) < optlen { + h.Options = make([]byte, optlen) + } else { + h.Options = h.Options[:optlen] + } + copy(h.Options, b[HeaderLen:hdrlen]) + } + return nil +} + +// ParseHeader parses b as an IPv4 header. +func ParseHeader(b []byte) (*Header, error) { + h := new(Header) + if err := h.Parse(b); err != nil { + return nil, err + } + return h, nil +} diff --git a/vendor/golang.org/x/net/ipv4/helper.go b/vendor/golang.org/x/net/ipv4/helper.go new file mode 100644 index 00000000..a5052e32 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/helper.go @@ -0,0 +1,63 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "errors" + "net" +) + +var ( + errMissingAddress = errors.New("missing address") + errMissingHeader = errors.New("missing header") + errHeaderTooShort = errors.New("header too short") + errBufferTooShort = errors.New("buffer too short") + errInvalidConnType = errors.New("invalid conn type") + errOpNoSupport = errors.New("operation not supported") + errNoSuchInterface = errors.New("no such interface") + errNoSuchMulticastInterface = errors.New("no such multicast interface") + + // See http://www.freebsd.org/doc/en/books/porters-handbook/freebsd-versions.html. + freebsdVersion uint32 +) + +func boolint(b bool) int { + if b { + return 1 + } + return 0 +} + +func netAddrToIP4(a net.Addr) net.IP { + switch v := a.(type) { + case *net.UDPAddr: + if ip := v.IP.To4(); ip != nil { + return ip + } + case *net.IPAddr: + if ip := v.IP.To4(); ip != nil { + return ip + } + } + return nil +} + +func opAddr(a net.Addr) net.Addr { + switch a.(type) { + case *net.TCPAddr: + if a == nil { + return nil + } + case *net.UDPAddr: + if a == nil { + return nil + } + case *net.IPAddr: + if a == nil { + return nil + } + } + return a +} diff --git a/vendor/golang.org/x/net/ipv4/iana.go b/vendor/golang.org/x/net/ipv4/iana.go new file mode 100644 index 00000000..4375b409 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/iana.go @@ -0,0 +1,38 @@ +// go generate gen.go +// Code generated by the command above; DO NOT EDIT. + +package ipv4 + +// Internet Control Message Protocol (ICMP) Parameters, Updated: 2018-02-26 +const ( + ICMPTypeEchoReply ICMPType = 0 // Echo Reply + ICMPTypeDestinationUnreachable ICMPType = 3 // Destination Unreachable + ICMPTypeRedirect ICMPType = 5 // Redirect + ICMPTypeEcho ICMPType = 8 // Echo + ICMPTypeRouterAdvertisement ICMPType = 9 // Router Advertisement + ICMPTypeRouterSolicitation ICMPType = 10 // Router Solicitation + ICMPTypeTimeExceeded ICMPType = 11 // Time Exceeded + ICMPTypeParameterProblem ICMPType = 12 // Parameter Problem + ICMPTypeTimestamp ICMPType = 13 // Timestamp + ICMPTypeTimestampReply ICMPType = 14 // Timestamp Reply + ICMPTypePhoturis ICMPType = 40 // Photuris + ICMPTypeExtendedEchoRequest ICMPType = 42 // Extended Echo Request + ICMPTypeExtendedEchoReply ICMPType = 43 // Extended Echo Reply +) + +// Internet Control Message Protocol (ICMP) Parameters, Updated: 2018-02-26 +var icmpTypes = map[ICMPType]string{ + 0: "echo reply", + 3: "destination unreachable", + 5: "redirect", + 8: "echo", + 9: "router advertisement", + 10: "router solicitation", + 11: "time exceeded", + 12: "parameter problem", + 13: "timestamp", + 14: "timestamp reply", + 40: "photuris", + 42: "extended echo request", + 43: "extended echo reply", +} diff --git a/vendor/golang.org/x/net/ipv4/icmp.go b/vendor/golang.org/x/net/ipv4/icmp.go new file mode 100644 index 00000000..9902bb3d --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/icmp.go @@ -0,0 +1,57 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import "golang.org/x/net/internal/iana" + +// An ICMPType represents a type of ICMP message. +type ICMPType int + +func (typ ICMPType) String() string { + s, ok := icmpTypes[typ] + if !ok { + return "" + } + return s +} + +// Protocol returns the ICMPv4 protocol number. +func (typ ICMPType) Protocol() int { + return iana.ProtocolICMP +} + +// An ICMPFilter represents an ICMP message filter for incoming +// packets. The filter belongs to a packet delivery path on a host and +// it cannot interact with forwarding packets or tunnel-outer packets. +// +// Note: RFC 8200 defines a reasonable role model and it works not +// only for IPv6 but IPv4. A node means a device that implements IP. +// A router means a node that forwards IP packets not explicitly +// addressed to itself, and a host means a node that is not a router. +type ICMPFilter struct { + icmpFilter +} + +// Accept accepts incoming ICMP packets including the type field value +// typ. +func (f *ICMPFilter) Accept(typ ICMPType) { + f.accept(typ) +} + +// Block blocks incoming ICMP packets including the type field value +// typ. +func (f *ICMPFilter) Block(typ ICMPType) { + f.block(typ) +} + +// SetAll sets the filter action to the filter. +func (f *ICMPFilter) SetAll(block bool) { + f.setAll(block) +} + +// WillBlock reports whether the ICMP type will be blocked. +func (f *ICMPFilter) WillBlock(typ ICMPType) bool { + return f.willBlock(typ) +} diff --git a/vendor/golang.org/x/net/ipv4/icmp_linux.go b/vendor/golang.org/x/net/ipv4/icmp_linux.go new file mode 100644 index 00000000..6e1c5c80 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/icmp_linux.go @@ -0,0 +1,25 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +func (f *icmpFilter) accept(typ ICMPType) { + f.Data &^= 1 << (uint32(typ) & 31) +} + +func (f *icmpFilter) block(typ ICMPType) { + f.Data |= 1 << (uint32(typ) & 31) +} + +func (f *icmpFilter) setAll(block bool) { + if block { + f.Data = 1<<32 - 1 + } else { + f.Data = 0 + } +} + +func (f *icmpFilter) willBlock(typ ICMPType) bool { + return f.Data&(1<<(uint32(typ)&31)) != 0 +} diff --git a/vendor/golang.org/x/net/ipv4/icmp_stub.go b/vendor/golang.org/x/net/ipv4/icmp_stub.go new file mode 100644 index 00000000..21bb29ab --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/icmp_stub.go @@ -0,0 +1,25 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux + +package ipv4 + +const sizeofICMPFilter = 0x0 + +type icmpFilter struct { +} + +func (f *icmpFilter) accept(typ ICMPType) { +} + +func (f *icmpFilter) block(typ ICMPType) { +} + +func (f *icmpFilter) setAll(block bool) { +} + +func (f *icmpFilter) willBlock(typ ICMPType) bool { + return false +} diff --git a/vendor/golang.org/x/net/ipv4/packet.go b/vendor/golang.org/x/net/ipv4/packet.go new file mode 100644 index 00000000..f00f5b05 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/packet.go @@ -0,0 +1,69 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + "syscall" + + "golang.org/x/net/internal/socket" +) + +// BUG(mikio): On Windows, the ReadFrom and WriteTo methods of RawConn +// are not implemented. + +// A packetHandler represents the IPv4 datagram handler. +type packetHandler struct { + *net.IPConn + *socket.Conn + rawOpt +} + +func (c *packetHandler) ok() bool { return c != nil && c.IPConn != nil && c.Conn != nil } + +// ReadFrom reads an IPv4 datagram from the endpoint c, copying the +// datagram into b. It returns the received datagram as the IPv4 +// header h, the payload p and the control message cm. +func (c *packetHandler) ReadFrom(b []byte) (h *Header, p []byte, cm *ControlMessage, err error) { + if !c.ok() { + return nil, nil, nil, syscall.EINVAL + } + return c.readFrom(b) +} + +func slicePacket(b []byte) (h, p []byte, err error) { + if len(b) < HeaderLen { + return nil, nil, errHeaderTooShort + } + hdrlen := int(b[0]&0x0f) << 2 + return b[:hdrlen], b[hdrlen:], nil +} + +// WriteTo writes an IPv4 datagram through the endpoint c, copying the +// datagram from the IPv4 header h and the payload p. The control +// message cm allows the datagram path and the outgoing interface to be +// specified. Currently only Darwin and Linux support this. The cm +// may be nil if control of the outgoing datagram is not required. +// +// The IPv4 header h must contain appropriate fields that include: +// +// Version = +// Len = +// TOS = +// TotalLen = +// ID = platform sets an appropriate value if ID is zero +// FragOff = +// TTL = +// Protocol = +// Checksum = platform sets an appropriate value if Checksum is zero +// Src = platform sets an appropriate value if Src is nil +// Dst = +// Options = optional +func (c *packetHandler) WriteTo(h *Header, p []byte, cm *ControlMessage) error { + if !c.ok() { + return syscall.EINVAL + } + return c.writeTo(h, p, cm) +} diff --git a/vendor/golang.org/x/net/ipv4/packet_go1_8.go b/vendor/golang.org/x/net/ipv4/packet_go1_8.go new file mode 100644 index 00000000..b47d1868 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/packet_go1_8.go @@ -0,0 +1,56 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.9 + +package ipv4 + +import "net" + +func (c *packetHandler) readFrom(b []byte) (h *Header, p []byte, cm *ControlMessage, err error) { + c.rawOpt.RLock() + oob := NewControlMessage(c.rawOpt.cflags) + c.rawOpt.RUnlock() + n, nn, _, src, err := c.ReadMsgIP(b, oob) + if err != nil { + return nil, nil, nil, err + } + var hs []byte + if hs, p, err = slicePacket(b[:n]); err != nil { + return nil, nil, nil, err + } + if h, err = ParseHeader(hs); err != nil { + return nil, nil, nil, err + } + if nn > 0 { + cm = new(ControlMessage) + if err := cm.Parse(oob[:nn]); err != nil { + return nil, nil, nil, err + } + } + if src != nil && cm != nil { + cm.Src = src.IP + } + return +} + +func (c *packetHandler) writeTo(h *Header, p []byte, cm *ControlMessage) error { + oob := cm.Marshal() + wh, err := h.Marshal() + if err != nil { + return err + } + dst := new(net.IPAddr) + if cm != nil { + if ip := cm.Dst.To4(); ip != nil { + dst.IP = ip + } + } + if dst.IP == nil { + dst.IP = h.Dst + } + wh = append(wh, p...) + _, _, err = c.WriteMsgIP(wh, oob, dst) + return err +} diff --git a/vendor/golang.org/x/net/ipv4/packet_go1_9.go b/vendor/golang.org/x/net/ipv4/packet_go1_9.go new file mode 100644 index 00000000..082c36d7 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/packet_go1_9.go @@ -0,0 +1,67 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +package ipv4 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +func (c *packetHandler) readFrom(b []byte) (h *Header, p []byte, cm *ControlMessage, err error) { + c.rawOpt.RLock() + m := socket.Message{ + Buffers: [][]byte{b}, + OOB: NewControlMessage(c.rawOpt.cflags), + } + c.rawOpt.RUnlock() + if err := c.RecvMsg(&m, 0); err != nil { + return nil, nil, nil, &net.OpError{Op: "read", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Err: err} + } + var hs []byte + if hs, p, err = slicePacket(b[:m.N]); err != nil { + return nil, nil, nil, &net.OpError{Op: "read", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Err: err} + } + if h, err = ParseHeader(hs); err != nil { + return nil, nil, nil, &net.OpError{Op: "read", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Err: err} + } + if m.NN > 0 { + cm = new(ControlMessage) + if err := cm.Parse(m.OOB[:m.NN]); err != nil { + return nil, nil, nil, &net.OpError{Op: "read", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Err: err} + } + } + if src, ok := m.Addr.(*net.IPAddr); ok && cm != nil { + cm.Src = src.IP + } + return +} + +func (c *packetHandler) writeTo(h *Header, p []byte, cm *ControlMessage) error { + m := socket.Message{ + OOB: cm.Marshal(), + } + wh, err := h.Marshal() + if err != nil { + return err + } + m.Buffers = [][]byte{wh, p} + dst := new(net.IPAddr) + if cm != nil { + if ip := cm.Dst.To4(); ip != nil { + dst.IP = ip + } + } + if dst.IP == nil { + dst.IP = h.Dst + } + m.Addr = dst + if err := c.SendMsg(&m, 0); err != nil { + return &net.OpError{Op: "write", Net: c.IPConn.LocalAddr().Network(), Source: c.IPConn.LocalAddr(), Addr: opAddr(dst), Err: err} + } + return nil +} diff --git a/vendor/golang.org/x/net/ipv4/payload.go b/vendor/golang.org/x/net/ipv4/payload.go new file mode 100644 index 00000000..f95f811a --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/payload.go @@ -0,0 +1,23 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +// BUG(mikio): On Windows, the ControlMessage for ReadFrom and WriteTo +// methods of PacketConn is not implemented. + +// A payloadHandler represents the IPv4 datagram payload handler. +type payloadHandler struct { + net.PacketConn + *socket.Conn + rawOpt +} + +func (c *payloadHandler) ok() bool { return c != nil && c.PacketConn != nil && c.Conn != nil } diff --git a/vendor/golang.org/x/net/ipv4/payload_cmsg.go b/vendor/golang.org/x/net/ipv4/payload_cmsg.go new file mode 100644 index 00000000..3f06d760 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/payload_cmsg.go @@ -0,0 +1,36 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !nacl,!plan9,!windows + +package ipv4 + +import ( + "net" + "syscall" +) + +// ReadFrom reads a payload of the received IPv4 datagram, from the +// endpoint c, copying the payload into b. It returns the number of +// bytes copied into b, the control message cm and the source address +// src of the received datagram. +func (c *payloadHandler) ReadFrom(b []byte) (n int, cm *ControlMessage, src net.Addr, err error) { + if !c.ok() { + return 0, nil, nil, syscall.EINVAL + } + return c.readFrom(b) +} + +// WriteTo writes a payload of the IPv4 datagram, to the destination +// address dst through the endpoint c, copying the payload from b. It +// returns the number of bytes written. The control message cm allows +// the datagram path and the outgoing interface to be specified. +// Currently only Darwin and Linux support this. The cm may be nil if +// control of the outgoing datagram is not required. +func (c *payloadHandler) WriteTo(b []byte, cm *ControlMessage, dst net.Addr) (n int, err error) { + if !c.ok() { + return 0, syscall.EINVAL + } + return c.writeTo(b, cm, dst) +} diff --git a/vendor/golang.org/x/net/ipv4/payload_cmsg_go1_8.go b/vendor/golang.org/x/net/ipv4/payload_cmsg_go1_8.go new file mode 100644 index 00000000..d26ccd90 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/payload_cmsg_go1_8.go @@ -0,0 +1,59 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.9 +// +build !nacl,!plan9,!windows + +package ipv4 + +import "net" + +func (c *payloadHandler) readFrom(b []byte) (n int, cm *ControlMessage, src net.Addr, err error) { + c.rawOpt.RLock() + oob := NewControlMessage(c.rawOpt.cflags) + c.rawOpt.RUnlock() + var nn int + switch c := c.PacketConn.(type) { + case *net.UDPConn: + if n, nn, _, src, err = c.ReadMsgUDP(b, oob); err != nil { + return 0, nil, nil, err + } + case *net.IPConn: + nb := make([]byte, maxHeaderLen+len(b)) + if n, nn, _, src, err = c.ReadMsgIP(nb, oob); err != nil { + return 0, nil, nil, err + } + hdrlen := int(nb[0]&0x0f) << 2 + copy(b, nb[hdrlen:]) + n -= hdrlen + default: + return 0, nil, nil, &net.OpError{Op: "read", Net: c.LocalAddr().Network(), Source: c.LocalAddr(), Err: errInvalidConnType} + } + if nn > 0 { + cm = new(ControlMessage) + if err = cm.Parse(oob[:nn]); err != nil { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + } + if cm != nil { + cm.Src = netAddrToIP4(src) + } + return +} + +func (c *payloadHandler) writeTo(b []byte, cm *ControlMessage, dst net.Addr) (n int, err error) { + oob := cm.Marshal() + if dst == nil { + return 0, &net.OpError{Op: "write", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: errMissingAddress} + } + switch c := c.PacketConn.(type) { + case *net.UDPConn: + n, _, err = c.WriteMsgUDP(b, oob, dst.(*net.UDPAddr)) + case *net.IPConn: + n, _, err = c.WriteMsgIP(b, oob, dst.(*net.IPAddr)) + default: + return 0, &net.OpError{Op: "write", Net: c.LocalAddr().Network(), Source: c.LocalAddr(), Addr: opAddr(dst), Err: errInvalidConnType} + } + return +} diff --git a/vendor/golang.org/x/net/ipv4/payload_cmsg_go1_9.go b/vendor/golang.org/x/net/ipv4/payload_cmsg_go1_9.go new file mode 100644 index 00000000..2f193118 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/payload_cmsg_go1_9.go @@ -0,0 +1,67 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 +// +build !nacl,!plan9,!windows + +package ipv4 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +func (c *payloadHandler) readFrom(b []byte) (int, *ControlMessage, net.Addr, error) { + c.rawOpt.RLock() + m := socket.Message{ + OOB: NewControlMessage(c.rawOpt.cflags), + } + c.rawOpt.RUnlock() + switch c.PacketConn.(type) { + case *net.UDPConn: + m.Buffers = [][]byte{b} + if err := c.RecvMsg(&m, 0); err != nil { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + case *net.IPConn: + h := make([]byte, HeaderLen) + m.Buffers = [][]byte{h, b} + if err := c.RecvMsg(&m, 0); err != nil { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + hdrlen := int(h[0]&0x0f) << 2 + if hdrlen > len(h) { + d := hdrlen - len(h) + copy(b, b[d:]) + m.N -= d + } else { + m.N -= hdrlen + } + default: + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: errInvalidConnType} + } + var cm *ControlMessage + if m.NN > 0 { + cm = new(ControlMessage) + if err := cm.Parse(m.OOB[:m.NN]); err != nil { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + cm.Src = netAddrToIP4(m.Addr) + } + return m.N, cm, m.Addr, nil +} + +func (c *payloadHandler) writeTo(b []byte, cm *ControlMessage, dst net.Addr) (int, error) { + m := socket.Message{ + Buffers: [][]byte{b}, + OOB: cm.Marshal(), + Addr: dst, + } + err := c.SendMsg(&m, 0) + if err != nil { + err = &net.OpError{Op: "write", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Addr: opAddr(dst), Err: err} + } + return m.N, err +} diff --git a/vendor/golang.org/x/net/ipv4/payload_nocmsg.go b/vendor/golang.org/x/net/ipv4/payload_nocmsg.go new file mode 100644 index 00000000..3926de70 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/payload_nocmsg.go @@ -0,0 +1,42 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build nacl plan9 windows + +package ipv4 + +import ( + "net" + "syscall" +) + +// ReadFrom reads a payload of the received IPv4 datagram, from the +// endpoint c, copying the payload into b. It returns the number of +// bytes copied into b, the control message cm and the source address +// src of the received datagram. +func (c *payloadHandler) ReadFrom(b []byte) (n int, cm *ControlMessage, src net.Addr, err error) { + if !c.ok() { + return 0, nil, nil, syscall.EINVAL + } + if n, src, err = c.PacketConn.ReadFrom(b); err != nil { + return 0, nil, nil, err + } + return +} + +// WriteTo writes a payload of the IPv4 datagram, to the destination +// address dst through the endpoint c, copying the payload from b. It +// returns the number of bytes written. The control message cm allows +// the datagram path and the outgoing interface to be specified. +// Currently only Darwin and Linux support this. The cm may be nil if +// control of the outgoing datagram is not required. +func (c *payloadHandler) WriteTo(b []byte, cm *ControlMessage, dst net.Addr) (n int, err error) { + if !c.ok() { + return 0, syscall.EINVAL + } + if dst == nil { + return 0, errMissingAddress + } + return c.PacketConn.WriteTo(b, dst) +} diff --git a/vendor/golang.org/x/net/ipv4/sockopt.go b/vendor/golang.org/x/net/ipv4/sockopt.go new file mode 100644 index 00000000..22e90c03 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sockopt.go @@ -0,0 +1,44 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import "golang.org/x/net/internal/socket" + +// Sticky socket options +const ( + ssoTOS = iota // header field for unicast packet + ssoTTL // header field for unicast packet + ssoMulticastTTL // header field for multicast packet + ssoMulticastInterface // outbound interface for multicast packet + ssoMulticastLoopback // loopback for multicast packet + ssoReceiveTTL // header field on received packet + ssoReceiveDst // header field on received packet + ssoReceiveInterface // inbound interface on received packet + ssoPacketInfo // incbound or outbound packet path + ssoHeaderPrepend // ipv4 header prepend + ssoStripHeader // strip ipv4 header + ssoICMPFilter // icmp filter + ssoJoinGroup // any-source multicast + ssoLeaveGroup // any-source multicast + ssoJoinSourceGroup // source-specific multicast + ssoLeaveSourceGroup // source-specific multicast + ssoBlockSourceGroup // any-source or source-specific multicast + ssoUnblockSourceGroup // any-source or source-specific multicast + ssoAttachFilter // attach BPF for filtering inbound traffic +) + +// Sticky socket option value types +const ( + ssoTypeIPMreq = iota + 1 + ssoTypeIPMreqn + ssoTypeGroupReq + ssoTypeGroupSourceReq +) + +// A sockOpt represents a binding for sticky socket option. +type sockOpt struct { + socket.Option + typ int // hint for option value type; optional +} diff --git a/vendor/golang.org/x/net/ipv4/sockopt_posix.go b/vendor/golang.org/x/net/ipv4/sockopt_posix.go new file mode 100644 index 00000000..e96955bc --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sockopt_posix.go @@ -0,0 +1,71 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris windows + +package ipv4 + +import ( + "net" + "unsafe" + + "golang.org/x/net/bpf" + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) getMulticastInterface(c *socket.Conn) (*net.Interface, error) { + switch so.typ { + case ssoTypeIPMreqn: + return so.getIPMreqn(c) + default: + return so.getMulticastIf(c) + } +} + +func (so *sockOpt) setMulticastInterface(c *socket.Conn, ifi *net.Interface) error { + switch so.typ { + case ssoTypeIPMreqn: + return so.setIPMreqn(c, ifi, nil) + default: + return so.setMulticastIf(c, ifi) + } +} + +func (so *sockOpt) getICMPFilter(c *socket.Conn) (*ICMPFilter, error) { + b := make([]byte, so.Len) + n, err := so.Get(c, b) + if err != nil { + return nil, err + } + if n != sizeofICMPFilter { + return nil, errOpNoSupport + } + return (*ICMPFilter)(unsafe.Pointer(&b[0])), nil +} + +func (so *sockOpt) setICMPFilter(c *socket.Conn, f *ICMPFilter) error { + b := (*[sizeofICMPFilter]byte)(unsafe.Pointer(f))[:sizeofICMPFilter] + return so.Set(c, b) +} + +func (so *sockOpt) setGroup(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + switch so.typ { + case ssoTypeIPMreq: + return so.setIPMreq(c, ifi, grp) + case ssoTypeIPMreqn: + return so.setIPMreqn(c, ifi, grp) + case ssoTypeGroupReq: + return so.setGroupReq(c, ifi, grp) + default: + return errOpNoSupport + } +} + +func (so *sockOpt) setSourceGroup(c *socket.Conn, ifi *net.Interface, grp, src net.IP) error { + return so.setGroupSourceReq(c, ifi, grp, src) +} + +func (so *sockOpt) setBPF(c *socket.Conn, f []bpf.RawInstruction) error { + return so.setAttachFilter(c, f) +} diff --git a/vendor/golang.org/x/net/ipv4/sockopt_stub.go b/vendor/golang.org/x/net/ipv4/sockopt_stub.go new file mode 100644 index 00000000..23249b78 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sockopt_stub.go @@ -0,0 +1,42 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package ipv4 + +import ( + "net" + + "golang.org/x/net/bpf" + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) getMulticastInterface(c *socket.Conn) (*net.Interface, error) { + return nil, errOpNoSupport +} + +func (so *sockOpt) setMulticastInterface(c *socket.Conn, ifi *net.Interface) error { + return errOpNoSupport +} + +func (so *sockOpt) getICMPFilter(c *socket.Conn) (*ICMPFilter, error) { + return nil, errOpNoSupport +} + +func (so *sockOpt) setICMPFilter(c *socket.Conn, f *ICMPFilter) error { + return errOpNoSupport +} + +func (so *sockOpt) setGroup(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + return errOpNoSupport +} + +func (so *sockOpt) setSourceGroup(c *socket.Conn, ifi *net.Interface, grp, src net.IP) error { + return errOpNoSupport +} + +func (so *sockOpt) setBPF(c *socket.Conn, f []bpf.RawInstruction) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv4/sys_asmreq.go b/vendor/golang.org/x/net/ipv4/sys_asmreq.go new file mode 100644 index 00000000..0388cba0 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_asmreq.go @@ -0,0 +1,119 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd solaris windows + +package ipv4 + +import ( + "net" + "unsafe" + + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setIPMreq(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + mreq := ipMreq{Multiaddr: [4]byte{grp[0], grp[1], grp[2], grp[3]}} + if err := setIPMreqInterface(&mreq, ifi); err != nil { + return err + } + b := (*[sizeofIPMreq]byte)(unsafe.Pointer(&mreq))[:sizeofIPMreq] + return so.Set(c, b) +} + +func (so *sockOpt) getMulticastIf(c *socket.Conn) (*net.Interface, error) { + var b [4]byte + if _, err := so.Get(c, b[:]); err != nil { + return nil, err + } + ifi, err := netIP4ToInterface(net.IPv4(b[0], b[1], b[2], b[3])) + if err != nil { + return nil, err + } + return ifi, nil +} + +func (so *sockOpt) setMulticastIf(c *socket.Conn, ifi *net.Interface) error { + ip, err := netInterfaceToIP4(ifi) + if err != nil { + return err + } + var b [4]byte + copy(b[:], ip) + return so.Set(c, b[:]) +} + +func setIPMreqInterface(mreq *ipMreq, ifi *net.Interface) error { + if ifi == nil { + return nil + } + ifat, err := ifi.Addrs() + if err != nil { + return err + } + for _, ifa := range ifat { + switch ifa := ifa.(type) { + case *net.IPAddr: + if ip := ifa.IP.To4(); ip != nil { + copy(mreq.Interface[:], ip) + return nil + } + case *net.IPNet: + if ip := ifa.IP.To4(); ip != nil { + copy(mreq.Interface[:], ip) + return nil + } + } + } + return errNoSuchInterface +} + +func netIP4ToInterface(ip net.IP) (*net.Interface, error) { + ift, err := net.Interfaces() + if err != nil { + return nil, err + } + for _, ifi := range ift { + ifat, err := ifi.Addrs() + if err != nil { + return nil, err + } + for _, ifa := range ifat { + switch ifa := ifa.(type) { + case *net.IPAddr: + if ip.Equal(ifa.IP) { + return &ifi, nil + } + case *net.IPNet: + if ip.Equal(ifa.IP) { + return &ifi, nil + } + } + } + } + return nil, errNoSuchInterface +} + +func netInterfaceToIP4(ifi *net.Interface) (net.IP, error) { + if ifi == nil { + return net.IPv4zero.To4(), nil + } + ifat, err := ifi.Addrs() + if err != nil { + return nil, err + } + for _, ifa := range ifat { + switch ifa := ifa.(type) { + case *net.IPAddr: + if ip := ifa.IP.To4(); ip != nil { + return ip, nil + } + case *net.IPNet: + if ip := ifa.IP.To4(); ip != nil { + return ip, nil + } + } + } + return nil, errNoSuchInterface +} diff --git a/vendor/golang.org/x/net/ipv4/sys_asmreq_stub.go b/vendor/golang.org/x/net/ipv4/sys_asmreq_stub.go new file mode 100644 index 00000000..f3919208 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_asmreq_stub.go @@ -0,0 +1,25 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!windows + +package ipv4 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setIPMreq(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + return errOpNoSupport +} + +func (so *sockOpt) getMulticastIf(c *socket.Conn) (*net.Interface, error) { + return nil, errOpNoSupport +} + +func (so *sockOpt) setMulticastIf(c *socket.Conn, ifi *net.Interface) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv4/sys_asmreqn.go b/vendor/golang.org/x/net/ipv4/sys_asmreqn.go new file mode 100644 index 00000000..1f24f69f --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_asmreqn.go @@ -0,0 +1,42 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux + +package ipv4 + +import ( + "net" + "unsafe" + + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) getIPMreqn(c *socket.Conn) (*net.Interface, error) { + b := make([]byte, so.Len) + if _, err := so.Get(c, b); err != nil { + return nil, err + } + mreqn := (*ipMreqn)(unsafe.Pointer(&b[0])) + if mreqn.Ifindex == 0 { + return nil, nil + } + ifi, err := net.InterfaceByIndex(int(mreqn.Ifindex)) + if err != nil { + return nil, err + } + return ifi, nil +} + +func (so *sockOpt) setIPMreqn(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + var mreqn ipMreqn + if ifi != nil { + mreqn.Ifindex = int32(ifi.Index) + } + if grp != nil { + mreqn.Multiaddr = [4]byte{grp[0], grp[1], grp[2], grp[3]} + } + b := (*[sizeofIPMreqn]byte)(unsafe.Pointer(&mreqn))[:sizeofIPMreqn] + return so.Set(c, b) +} diff --git a/vendor/golang.org/x/net/ipv4/sys_asmreqn_stub.go b/vendor/golang.org/x/net/ipv4/sys_asmreqn_stub.go new file mode 100644 index 00000000..0711d3d7 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_asmreqn_stub.go @@ -0,0 +1,21 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!freebsd,!linux + +package ipv4 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) getIPMreqn(c *socket.Conn) (*net.Interface, error) { + return nil, errOpNoSupport +} + +func (so *sockOpt) setIPMreqn(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv4/sys_bpf.go b/vendor/golang.org/x/net/ipv4/sys_bpf.go new file mode 100644 index 00000000..9f30b730 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_bpf.go @@ -0,0 +1,23 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package ipv4 + +import ( + "unsafe" + + "golang.org/x/net/bpf" + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setAttachFilter(c *socket.Conn, f []bpf.RawInstruction) error { + prog := sockFProg{ + Len: uint16(len(f)), + Filter: (*sockFilter)(unsafe.Pointer(&f[0])), + } + b := (*[sizeofSockFprog]byte)(unsafe.Pointer(&prog))[:sizeofSockFprog] + return so.Set(c, b) +} diff --git a/vendor/golang.org/x/net/ipv4/sys_bpf_stub.go b/vendor/golang.org/x/net/ipv4/sys_bpf_stub.go new file mode 100644 index 00000000..9a213209 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_bpf_stub.go @@ -0,0 +1,16 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux + +package ipv4 + +import ( + "golang.org/x/net/bpf" + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setAttachFilter(c *socket.Conn, f []bpf.RawInstruction) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv4/sys_bsd.go b/vendor/golang.org/x/net/ipv4/sys_bsd.go new file mode 100644 index 00000000..58256dd9 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_bsd.go @@ -0,0 +1,37 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build netbsd openbsd + +package ipv4 + +import ( + "net" + "syscall" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTTL: {sysIP_RECVTTL, 1, marshalTTL, parseTTL}, + ctlDst: {sysIP_RECVDSTADDR, net.IPv4len, marshalDst, parseDst}, + ctlInterface: {sysIP_RECVIF, syscall.SizeofSockaddrDatalink, marshalInterface, parseInterface}, + } + + sockOpts = map[int]*sockOpt{ + ssoTOS: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TOS, Len: 4}}, + ssoTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TTL, Len: 4}}, + ssoMulticastTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_TTL, Len: 1}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_LOOP, Len: 1}}, + ssoReceiveTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVTTL, Len: 4}}, + ssoReceiveDst: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVDSTADDR, Len: 4}}, + ssoReceiveInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVIF, Len: 4}}, + ssoHeaderPrepend: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_HDRINCL, Len: 4}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_ADD_MEMBERSHIP, Len: sizeofIPMreq}, typ: ssoTypeIPMreq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_DROP_MEMBERSHIP, Len: sizeofIPMreq}, typ: ssoTypeIPMreq}, + } +) diff --git a/vendor/golang.org/x/net/ipv4/sys_darwin.go b/vendor/golang.org/x/net/ipv4/sys_darwin.go new file mode 100644 index 00000000..e8fb1916 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_darwin.go @@ -0,0 +1,93 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + "strconv" + "strings" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTTL: {sysIP_RECVTTL, 1, marshalTTL, parseTTL}, + ctlDst: {sysIP_RECVDSTADDR, net.IPv4len, marshalDst, parseDst}, + ctlInterface: {sysIP_RECVIF, syscall.SizeofSockaddrDatalink, marshalInterface, parseInterface}, + } + + sockOpts = map[int]*sockOpt{ + ssoTOS: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TOS, Len: 4}}, + ssoTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TTL, Len: 4}}, + ssoMulticastTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_TTL, Len: 1}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_LOOP, Len: 4}}, + ssoReceiveTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVTTL, Len: 4}}, + ssoReceiveDst: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVDSTADDR, Len: 4}}, + ssoReceiveInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVIF, Len: 4}}, + ssoHeaderPrepend: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_HDRINCL, Len: 4}}, + ssoStripHeader: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_STRIPHDR, Len: 4}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_ADD_MEMBERSHIP, Len: sizeofIPMreq}, typ: ssoTypeIPMreq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_DROP_MEMBERSHIP, Len: sizeofIPMreq}, typ: ssoTypeIPMreq}, + } +) + +func init() { + // Seems like kern.osreldate is veiled on latest OS X. We use + // kern.osrelease instead. + s, err := syscall.Sysctl("kern.osrelease") + if err != nil { + return + } + ss := strings.Split(s, ".") + if len(ss) == 0 { + return + } + // The IP_PKTINFO and protocol-independent multicast API were + // introduced in OS X 10.7 (Darwin 11). But it looks like + // those features require OS X 10.8 (Darwin 12) or above. + // See http://support.apple.com/kb/HT1633. + if mjver, err := strconv.Atoi(ss[0]); err != nil || mjver < 12 { + return + } + ctlOpts[ctlPacketInfo].name = sysIP_PKTINFO + ctlOpts[ctlPacketInfo].length = sizeofInetPktinfo + ctlOpts[ctlPacketInfo].marshal = marshalPacketInfo + ctlOpts[ctlPacketInfo].parse = parsePacketInfo + sockOpts[ssoPacketInfo] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVPKTINFO, Len: 4}} + sockOpts[ssoMulticastInterface] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: sizeofIPMreqn}, typ: ssoTypeIPMreqn} + sockOpts[ssoJoinGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_JOIN_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq} + sockOpts[ssoLeaveGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_LEAVE_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq} + sockOpts[ssoJoinSourceGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_JOIN_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq} + sockOpts[ssoLeaveSourceGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_LEAVE_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq} + sockOpts[ssoBlockSourceGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_BLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq} + sockOpts[ssoUnblockSourceGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_UNBLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq} +} + +func (pi *inetPktinfo) setIfindex(i int) { + pi.Ifindex = uint32(i) +} + +func (gr *groupReq) setGroup(grp net.IP) { + sa := (*sockaddrInet)(unsafe.Pointer(uintptr(unsafe.Pointer(gr)) + 4)) + sa.Len = sizeofSockaddrInet + sa.Family = syscall.AF_INET + copy(sa.Addr[:], grp) +} + +func (gsr *groupSourceReq) setSourceGroup(grp, src net.IP) { + sa := (*sockaddrInet)(unsafe.Pointer(uintptr(unsafe.Pointer(gsr)) + 4)) + sa.Len = sizeofSockaddrInet + sa.Family = syscall.AF_INET + copy(sa.Addr[:], grp) + sa = (*sockaddrInet)(unsafe.Pointer(uintptr(unsafe.Pointer(gsr)) + 132)) + sa.Len = sizeofSockaddrInet + sa.Family = syscall.AF_INET + copy(sa.Addr[:], src) +} diff --git a/vendor/golang.org/x/net/ipv4/sys_dragonfly.go b/vendor/golang.org/x/net/ipv4/sys_dragonfly.go new file mode 100644 index 00000000..859764f3 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_dragonfly.go @@ -0,0 +1,35 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + "syscall" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTTL: {sysIP_RECVTTL, 1, marshalTTL, parseTTL}, + ctlDst: {sysIP_RECVDSTADDR, net.IPv4len, marshalDst, parseDst}, + ctlInterface: {sysIP_RECVIF, syscall.SizeofSockaddrDatalink, marshalInterface, parseInterface}, + } + + sockOpts = map[int]*sockOpt{ + ssoTOS: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TOS, Len: 4}}, + ssoTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TTL, Len: 4}}, + ssoMulticastTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_TTL, Len: 1}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_LOOP, Len: 4}}, + ssoReceiveTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVTTL, Len: 4}}, + ssoReceiveDst: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVDSTADDR, Len: 4}}, + ssoReceiveInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVIF, Len: 4}}, + ssoHeaderPrepend: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_HDRINCL, Len: 4}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_ADD_MEMBERSHIP, Len: sizeofIPMreq}, typ: ssoTypeIPMreq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_DROP_MEMBERSHIP, Len: sizeofIPMreq}, typ: ssoTypeIPMreq}, + } +) diff --git a/vendor/golang.org/x/net/ipv4/sys_freebsd.go b/vendor/golang.org/x/net/ipv4/sys_freebsd.go new file mode 100644 index 00000000..b8003245 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_freebsd.go @@ -0,0 +1,76 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + "runtime" + "strings" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTTL: {sysIP_RECVTTL, 1, marshalTTL, parseTTL}, + ctlDst: {sysIP_RECVDSTADDR, net.IPv4len, marshalDst, parseDst}, + ctlInterface: {sysIP_RECVIF, syscall.SizeofSockaddrDatalink, marshalInterface, parseInterface}, + } + + sockOpts = map[int]*sockOpt{ + ssoTOS: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TOS, Len: 4}}, + ssoTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TTL, Len: 4}}, + ssoMulticastTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_TTL, Len: 1}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_LOOP, Len: 4}}, + ssoReceiveTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVTTL, Len: 4}}, + ssoReceiveDst: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVDSTADDR, Len: 4}}, + ssoReceiveInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVIF, Len: 4}}, + ssoHeaderPrepend: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_HDRINCL, Len: 4}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_JOIN_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_LEAVE_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoJoinSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_JOIN_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoLeaveSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_LEAVE_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoBlockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_BLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoUnblockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_UNBLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + } +) + +func init() { + freebsdVersion, _ = syscall.SysctlUint32("kern.osreldate") + if freebsdVersion >= 1000000 { + sockOpts[ssoMulticastInterface] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: sizeofIPMreqn}, typ: ssoTypeIPMreqn} + } + if runtime.GOOS == "freebsd" && runtime.GOARCH == "386" { + archs, _ := syscall.Sysctl("kern.supported_archs") + for _, s := range strings.Fields(archs) { + if s == "amd64" { + freebsd32o64 = true + break + } + } + } +} + +func (gr *groupReq) setGroup(grp net.IP) { + sa := (*sockaddrInet)(unsafe.Pointer(&gr.Group)) + sa.Len = sizeofSockaddrInet + sa.Family = syscall.AF_INET + copy(sa.Addr[:], grp) +} + +func (gsr *groupSourceReq) setSourceGroup(grp, src net.IP) { + sa := (*sockaddrInet)(unsafe.Pointer(&gsr.Group)) + sa.Len = sizeofSockaddrInet + sa.Family = syscall.AF_INET + copy(sa.Addr[:], grp) + sa = (*sockaddrInet)(unsafe.Pointer(&gsr.Source)) + sa.Len = sizeofSockaddrInet + sa.Family = syscall.AF_INET + copy(sa.Addr[:], src) +} diff --git a/vendor/golang.org/x/net/ipv4/sys_linux.go b/vendor/golang.org/x/net/ipv4/sys_linux.go new file mode 100644 index 00000000..60defe13 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_linux.go @@ -0,0 +1,59 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTTL: {sysIP_TTL, 1, marshalTTL, parseTTL}, + ctlPacketInfo: {sysIP_PKTINFO, sizeofInetPktinfo, marshalPacketInfo, parsePacketInfo}, + } + + sockOpts = map[int]*sockOpt{ + ssoTOS: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TOS, Len: 4}}, + ssoTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TTL, Len: 4}}, + ssoMulticastTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_TTL, Len: 4}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: sizeofIPMreqn}, typ: ssoTypeIPMreqn}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_LOOP, Len: 4}}, + ssoReceiveTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVTTL, Len: 4}}, + ssoPacketInfo: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_PKTINFO, Len: 4}}, + ssoHeaderPrepend: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_HDRINCL, Len: 4}}, + ssoICMPFilter: {Option: socket.Option{Level: iana.ProtocolReserved, Name: sysICMP_FILTER, Len: sizeofICMPFilter}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_JOIN_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_LEAVE_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoJoinSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_JOIN_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoLeaveSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_LEAVE_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoBlockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_BLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoUnblockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_UNBLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoAttachFilter: {Option: socket.Option{Level: sysSOL_SOCKET, Name: sysSO_ATTACH_FILTER, Len: sizeofSockFprog}}, + } +) + +func (pi *inetPktinfo) setIfindex(i int) { + pi.Ifindex = int32(i) +} + +func (gr *groupReq) setGroup(grp net.IP) { + sa := (*sockaddrInet)(unsafe.Pointer(&gr.Group)) + sa.Family = syscall.AF_INET + copy(sa.Addr[:], grp) +} + +func (gsr *groupSourceReq) setSourceGroup(grp, src net.IP) { + sa := (*sockaddrInet)(unsafe.Pointer(&gsr.Group)) + sa.Family = syscall.AF_INET + copy(sa.Addr[:], grp) + sa = (*sockaddrInet)(unsafe.Pointer(&gsr.Source)) + sa.Family = syscall.AF_INET + copy(sa.Addr[:], src) +} diff --git a/vendor/golang.org/x/net/ipv4/sys_solaris.go b/vendor/golang.org/x/net/ipv4/sys_solaris.go new file mode 100644 index 00000000..832fef1e --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_solaris.go @@ -0,0 +1,57 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "net" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTTL: {sysIP_RECVTTL, 4, marshalTTL, parseTTL}, + ctlPacketInfo: {sysIP_PKTINFO, sizeofInetPktinfo, marshalPacketInfo, parsePacketInfo}, + } + + sockOpts = map[int]sockOpt{ + ssoTOS: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TOS, Len: 4}}, + ssoTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TTL, Len: 4}}, + ssoMulticastTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_TTL, Len: 1}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_LOOP, Len: 1}}, + ssoReceiveTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVTTL, Len: 4}}, + ssoPacketInfo: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_RECVPKTINFO, Len: 4}}, + ssoHeaderPrepend: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_HDRINCL, Len: 4}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_JOIN_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_LEAVE_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoJoinSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_JOIN_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoLeaveSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_LEAVE_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoBlockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_BLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoUnblockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysMCAST_UNBLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + } +) + +func (pi *inetPktinfo) setIfindex(i int) { + pi.Ifindex = uint32(i) +} + +func (gr *groupReq) setGroup(grp net.IP) { + sa := (*sockaddrInet)(unsafe.Pointer(uintptr(unsafe.Pointer(gr)) + 4)) + sa.Family = syscall.AF_INET + copy(sa.Addr[:], grp) +} + +func (gsr *groupSourceReq) setSourceGroup(grp, src net.IP) { + sa := (*sockaddrInet)(unsafe.Pointer(uintptr(unsafe.Pointer(gsr)) + 4)) + sa.Family = syscall.AF_INET + copy(sa.Addr[:], grp) + sa = (*sockaddrInet)(unsafe.Pointer(uintptr(unsafe.Pointer(gsr)) + 260)) + sa.Family = syscall.AF_INET + copy(sa.Addr[:], src) +} diff --git a/vendor/golang.org/x/net/ipv4/sys_ssmreq.go b/vendor/golang.org/x/net/ipv4/sys_ssmreq.go new file mode 100644 index 00000000..ae5704e7 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_ssmreq.go @@ -0,0 +1,54 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux solaris + +package ipv4 + +import ( + "net" + "unsafe" + + "golang.org/x/net/internal/socket" +) + +var freebsd32o64 bool + +func (so *sockOpt) setGroupReq(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + var gr groupReq + if ifi != nil { + gr.Interface = uint32(ifi.Index) + } + gr.setGroup(grp) + var b []byte + if freebsd32o64 { + var d [sizeofGroupReq + 4]byte + s := (*[sizeofGroupReq]byte)(unsafe.Pointer(&gr)) + copy(d[:4], s[:4]) + copy(d[8:], s[4:]) + b = d[:] + } else { + b = (*[sizeofGroupReq]byte)(unsafe.Pointer(&gr))[:sizeofGroupReq] + } + return so.Set(c, b) +} + +func (so *sockOpt) setGroupSourceReq(c *socket.Conn, ifi *net.Interface, grp, src net.IP) error { + var gsr groupSourceReq + if ifi != nil { + gsr.Interface = uint32(ifi.Index) + } + gsr.setSourceGroup(grp, src) + var b []byte + if freebsd32o64 { + var d [sizeofGroupSourceReq + 4]byte + s := (*[sizeofGroupSourceReq]byte)(unsafe.Pointer(&gsr)) + copy(d[:4], s[:4]) + copy(d[8:], s[4:]) + b = d[:] + } else { + b = (*[sizeofGroupSourceReq]byte)(unsafe.Pointer(&gsr))[:sizeofGroupSourceReq] + } + return so.Set(c, b) +} diff --git a/vendor/golang.org/x/net/ipv4/sys_ssmreq_stub.go b/vendor/golang.org/x/net/ipv4/sys_ssmreq_stub.go new file mode 100644 index 00000000..e6b7623d --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_ssmreq_stub.go @@ -0,0 +1,21 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!freebsd,!linux,!solaris + +package ipv4 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setGroupReq(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + return errOpNoSupport +} + +func (so *sockOpt) setGroupSourceReq(c *socket.Conn, ifi *net.Interface, grp, src net.IP) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/sys/unix/zsysnum_solaris_amd64.go b/vendor/golang.org/x/net/ipv4/sys_stub.go similarity index 50% rename from vendor/golang.org/x/sys/unix/zsysnum_solaris_amd64.go rename to vendor/golang.org/x/net/ipv4/sys_stub.go index c7086598..4f076473 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_solaris_amd64.go +++ b/vendor/golang.org/x/net/ipv4/sys_stub.go @@ -2,12 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build amd64,solaris +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows -package unix +package ipv4 -// TODO(aram): remove these before Go 1.3. -const ( - SYS_EXECVE = 59 - SYS_FCNTL = 62 +var ( + ctlOpts = [ctlMax]ctlOpt{} + + sockOpts = map[int]*sockOpt{} ) diff --git a/vendor/golang.org/x/net/ipv4/sys_windows.go b/vendor/golang.org/x/net/ipv4/sys_windows.go new file mode 100644 index 00000000..b0913d53 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/sys_windows.go @@ -0,0 +1,67 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv4 + +import ( + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +const ( + // See ws2tcpip.h. + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_MULTICAST_IF = 0x9 + sysIP_MULTICAST_TTL = 0xa + sysIP_MULTICAST_LOOP = 0xb + sysIP_ADD_MEMBERSHIP = 0xc + sysIP_DROP_MEMBERSHIP = 0xd + sysIP_DONTFRAGMENT = 0xe + sysIP_ADD_SOURCE_MEMBERSHIP = 0xf + sysIP_DROP_SOURCE_MEMBERSHIP = 0x10 + sysIP_PKTINFO = 0x13 + + sizeofInetPktinfo = 0x8 + sizeofIPMreq = 0x8 + sizeofIPMreqSource = 0xc +) + +type inetPktinfo struct { + Addr [4]byte + Ifindex int32 +} + +type ipMreq struct { + Multiaddr [4]byte + Interface [4]byte +} + +type ipMreqSource struct { + Multiaddr [4]byte + Sourceaddr [4]byte + Interface [4]byte +} + +// See http://msdn.microsoft.com/en-us/library/windows/desktop/ms738586(v=vs.85).aspx +var ( + ctlOpts = [ctlMax]ctlOpt{} + + sockOpts = map[int]*sockOpt{ + ssoTOS: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TOS, Len: 4}}, + ssoTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_TTL, Len: 4}}, + ssoMulticastTTL: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_TTL, Len: 4}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_IF, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_MULTICAST_LOOP, Len: 4}}, + ssoHeaderPrepend: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_HDRINCL, Len: 4}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_ADD_MEMBERSHIP, Len: sizeofIPMreq}, typ: ssoTypeIPMreq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIP, Name: sysIP_DROP_MEMBERSHIP, Len: sizeofIPMreq}, typ: ssoTypeIPMreq}, + } +) + +func (pi *inetPktinfo) setIfindex(i int) { + pi.Ifindex = int32(i) +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_darwin.go b/vendor/golang.org/x/net/ipv4/zsys_darwin.go new file mode 100644 index 00000000..c07cc883 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_darwin.go @@ -0,0 +1,99 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_darwin.go + +package ipv4 + +const ( + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_RECVOPTS = 0x5 + sysIP_RECVRETOPTS = 0x6 + sysIP_RECVDSTADDR = 0x7 + sysIP_RETOPTS = 0x8 + sysIP_RECVIF = 0x14 + sysIP_STRIPHDR = 0x17 + sysIP_RECVTTL = 0x18 + sysIP_BOUND_IF = 0x19 + sysIP_PKTINFO = 0x1a + sysIP_RECVPKTINFO = 0x1a + + sysIP_MULTICAST_IF = 0x9 + sysIP_MULTICAST_TTL = 0xa + sysIP_MULTICAST_LOOP = 0xb + sysIP_ADD_MEMBERSHIP = 0xc + sysIP_DROP_MEMBERSHIP = 0xd + sysIP_MULTICAST_VIF = 0xe + sysIP_MULTICAST_IFINDEX = 0x42 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x46 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x47 + sysIP_BLOCK_SOURCE = 0x48 + sysIP_UNBLOCK_SOURCE = 0x49 + sysMCAST_JOIN_GROUP = 0x50 + sysMCAST_LEAVE_GROUP = 0x51 + sysMCAST_JOIN_SOURCE_GROUP = 0x52 + sysMCAST_LEAVE_SOURCE_GROUP = 0x53 + sysMCAST_BLOCK_SOURCE = 0x54 + sysMCAST_UNBLOCK_SOURCE = 0x55 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 +) + +type sockaddrStorage struct { + Len uint8 + Family uint8 + X__ss_pad1 [6]int8 + X__ss_align int64 + X__ss_pad2 [112]int8 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type inetPktinfo struct { + Ifindex uint32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr [4]byte /* in_addr */ + Sourceaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [128]byte +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [128]byte + Pad_cgo_1 [128]byte +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_dragonfly.go b/vendor/golang.org/x/net/ipv4/zsys_dragonfly.go new file mode 100644 index 00000000..c4365e9e --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_dragonfly.go @@ -0,0 +1,31 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_dragonfly.go + +package ipv4 + +const ( + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_RECVOPTS = 0x5 + sysIP_RECVRETOPTS = 0x6 + sysIP_RECVDSTADDR = 0x7 + sysIP_RETOPTS = 0x8 + sysIP_RECVIF = 0x14 + sysIP_RECVTTL = 0x41 + + sysIP_MULTICAST_IF = 0x9 + sysIP_MULTICAST_TTL = 0xa + sysIP_MULTICAST_LOOP = 0xb + sysIP_MULTICAST_VIF = 0xe + sysIP_ADD_MEMBERSHIP = 0xc + sysIP_DROP_MEMBERSHIP = 0xd + + sizeofIPMreq = 0x8 +) + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_freebsd_386.go b/vendor/golang.org/x/net/ipv4/zsys_freebsd_386.go new file mode 100644 index 00000000..8c4aec94 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_freebsd_386.go @@ -0,0 +1,93 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package ipv4 + +const ( + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_RECVOPTS = 0x5 + sysIP_RECVRETOPTS = 0x6 + sysIP_RECVDSTADDR = 0x7 + sysIP_SENDSRCADDR = 0x7 + sysIP_RETOPTS = 0x8 + sysIP_RECVIF = 0x14 + sysIP_ONESBCAST = 0x17 + sysIP_BINDANY = 0x18 + sysIP_RECVTTL = 0x41 + sysIP_MINTTL = 0x42 + sysIP_DONTFRAG = 0x43 + sysIP_RECVTOS = 0x44 + + sysIP_MULTICAST_IF = 0x9 + sysIP_MULTICAST_TTL = 0xa + sysIP_MULTICAST_LOOP = 0xb + sysIP_ADD_MEMBERSHIP = 0xc + sysIP_DROP_MEMBERSHIP = 0xd + sysIP_MULTICAST_VIF = 0xe + sysIP_ADD_SOURCE_MEMBERSHIP = 0x46 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x47 + sysIP_BLOCK_SOURCE = 0x48 + sysIP_UNBLOCK_SOURCE = 0x49 + sysMCAST_JOIN_GROUP = 0x50 + sysMCAST_LEAVE_GROUP = 0x51 + sysMCAST_JOIN_SOURCE_GROUP = 0x52 + sysMCAST_LEAVE_SOURCE_GROUP = 0x53 + sysMCAST_BLOCK_SOURCE = 0x54 + sysMCAST_UNBLOCK_SOURCE = 0x55 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 +) + +type sockaddrStorage struct { + Len uint8 + Family uint8 + X__ss_pad1 [6]int8 + X__ss_align int64 + X__ss_pad2 [112]int8 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr [4]byte /* in_addr */ + Sourceaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type groupReq struct { + Interface uint32 + Group sockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group sockaddrStorage + Source sockaddrStorage +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_freebsd_amd64.go b/vendor/golang.org/x/net/ipv4/zsys_freebsd_amd64.go new file mode 100644 index 00000000..4b10b7c5 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_freebsd_amd64.go @@ -0,0 +1,95 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package ipv4 + +const ( + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_RECVOPTS = 0x5 + sysIP_RECVRETOPTS = 0x6 + sysIP_RECVDSTADDR = 0x7 + sysIP_SENDSRCADDR = 0x7 + sysIP_RETOPTS = 0x8 + sysIP_RECVIF = 0x14 + sysIP_ONESBCAST = 0x17 + sysIP_BINDANY = 0x18 + sysIP_RECVTTL = 0x41 + sysIP_MINTTL = 0x42 + sysIP_DONTFRAG = 0x43 + sysIP_RECVTOS = 0x44 + + sysIP_MULTICAST_IF = 0x9 + sysIP_MULTICAST_TTL = 0xa + sysIP_MULTICAST_LOOP = 0xb + sysIP_ADD_MEMBERSHIP = 0xc + sysIP_DROP_MEMBERSHIP = 0xd + sysIP_MULTICAST_VIF = 0xe + sysIP_ADD_SOURCE_MEMBERSHIP = 0x46 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x47 + sysIP_BLOCK_SOURCE = 0x48 + sysIP_UNBLOCK_SOURCE = 0x49 + sysMCAST_JOIN_GROUP = 0x50 + sysMCAST_LEAVE_GROUP = 0x51 + sysMCAST_JOIN_SOURCE_GROUP = 0x52 + sysMCAST_LEAVE_SOURCE_GROUP = 0x53 + sysMCAST_BLOCK_SOURCE = 0x54 + sysMCAST_UNBLOCK_SOURCE = 0x55 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 +) + +type sockaddrStorage struct { + Len uint8 + Family uint8 + X__ss_pad1 [6]int8 + X__ss_align int64 + X__ss_pad2 [112]int8 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr [4]byte /* in_addr */ + Sourceaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group sockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group sockaddrStorage + Source sockaddrStorage +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_freebsd_arm.go b/vendor/golang.org/x/net/ipv4/zsys_freebsd_arm.go new file mode 100644 index 00000000..4b10b7c5 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_freebsd_arm.go @@ -0,0 +1,95 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package ipv4 + +const ( + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_RECVOPTS = 0x5 + sysIP_RECVRETOPTS = 0x6 + sysIP_RECVDSTADDR = 0x7 + sysIP_SENDSRCADDR = 0x7 + sysIP_RETOPTS = 0x8 + sysIP_RECVIF = 0x14 + sysIP_ONESBCAST = 0x17 + sysIP_BINDANY = 0x18 + sysIP_RECVTTL = 0x41 + sysIP_MINTTL = 0x42 + sysIP_DONTFRAG = 0x43 + sysIP_RECVTOS = 0x44 + + sysIP_MULTICAST_IF = 0x9 + sysIP_MULTICAST_TTL = 0xa + sysIP_MULTICAST_LOOP = 0xb + sysIP_ADD_MEMBERSHIP = 0xc + sysIP_DROP_MEMBERSHIP = 0xd + sysIP_MULTICAST_VIF = 0xe + sysIP_ADD_SOURCE_MEMBERSHIP = 0x46 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x47 + sysIP_BLOCK_SOURCE = 0x48 + sysIP_UNBLOCK_SOURCE = 0x49 + sysMCAST_JOIN_GROUP = 0x50 + sysMCAST_LEAVE_GROUP = 0x51 + sysMCAST_JOIN_SOURCE_GROUP = 0x52 + sysMCAST_LEAVE_SOURCE_GROUP = 0x53 + sysMCAST_BLOCK_SOURCE = 0x54 + sysMCAST_UNBLOCK_SOURCE = 0x55 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 +) + +type sockaddrStorage struct { + Len uint8 + Family uint8 + X__ss_pad1 [6]int8 + X__ss_align int64 + X__ss_pad2 [112]int8 +} + +type sockaddrInet struct { + Len uint8 + Family uint8 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr [4]byte /* in_addr */ + Sourceaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group sockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group sockaddrStorage + Source sockaddrStorage +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_386.go b/vendor/golang.org/x/net/ipv4/zsys_linux_386.go new file mode 100644 index 00000000..c0260f0c --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_386.go @@ -0,0 +1,148 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_amd64.go b/vendor/golang.org/x/net/ipv4/zsys_linux_amd64.go new file mode 100644 index 00000000..9c967eaa --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_amd64.go @@ -0,0 +1,150 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_arm.go b/vendor/golang.org/x/net/ipv4/zsys_linux_arm.go new file mode 100644 index 00000000..c0260f0c --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_arm.go @@ -0,0 +1,148 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_arm64.go b/vendor/golang.org/x/net/ipv4/zsys_linux_arm64.go new file mode 100644 index 00000000..9c967eaa --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_arm64.go @@ -0,0 +1,150 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_mips.go b/vendor/golang.org/x/net/ipv4/zsys_linux_mips.go new file mode 100644 index 00000000..c0260f0c --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_mips.go @@ -0,0 +1,148 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_mips64.go b/vendor/golang.org/x/net/ipv4/zsys_linux_mips64.go new file mode 100644 index 00000000..9c967eaa --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_mips64.go @@ -0,0 +1,150 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_mips64le.go b/vendor/golang.org/x/net/ipv4/zsys_linux_mips64le.go new file mode 100644 index 00000000..9c967eaa --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_mips64le.go @@ -0,0 +1,150 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_mipsle.go b/vendor/golang.org/x/net/ipv4/zsys_linux_mipsle.go new file mode 100644 index 00000000..c0260f0c --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_mipsle.go @@ -0,0 +1,148 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_ppc.go b/vendor/golang.org/x/net/ipv4/zsys_linux_ppc.go new file mode 100644 index 00000000..f65bd9a7 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_ppc.go @@ -0,0 +1,148 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]uint8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_ppc64.go b/vendor/golang.org/x/net/ipv4/zsys_linux_ppc64.go new file mode 100644 index 00000000..9c967eaa --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_ppc64.go @@ -0,0 +1,150 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_ppc64le.go b/vendor/golang.org/x/net/ipv4/zsys_linux_ppc64le.go new file mode 100644 index 00000000..9c967eaa --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_ppc64le.go @@ -0,0 +1,150 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_linux_s390x.go b/vendor/golang.org/x/net/ipv4/zsys_linux_s390x.go new file mode 100644 index 00000000..9c967eaa --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_linux_s390x.go @@ -0,0 +1,150 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv4 + +const ( + sysIP_TOS = 0x1 + sysIP_TTL = 0x2 + sysIP_HDRINCL = 0x3 + sysIP_OPTIONS = 0x4 + sysIP_ROUTER_ALERT = 0x5 + sysIP_RECVOPTS = 0x6 + sysIP_RETOPTS = 0x7 + sysIP_PKTINFO = 0x8 + sysIP_PKTOPTIONS = 0x9 + sysIP_MTU_DISCOVER = 0xa + sysIP_RECVERR = 0xb + sysIP_RECVTTL = 0xc + sysIP_RECVTOS = 0xd + sysIP_MTU = 0xe + sysIP_FREEBIND = 0xf + sysIP_TRANSPARENT = 0x13 + sysIP_RECVRETOPTS = 0x7 + sysIP_ORIGDSTADDR = 0x14 + sysIP_RECVORIGDSTADDR = 0x14 + sysIP_MINTTL = 0x15 + sysIP_NODEFRAG = 0x16 + sysIP_UNICAST_IF = 0x32 + + sysIP_MULTICAST_IF = 0x20 + sysIP_MULTICAST_TTL = 0x21 + sysIP_MULTICAST_LOOP = 0x22 + sysIP_ADD_MEMBERSHIP = 0x23 + sysIP_DROP_MEMBERSHIP = 0x24 + sysIP_UNBLOCK_SOURCE = 0x25 + sysIP_BLOCK_SOURCE = 0x26 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x27 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x28 + sysIP_MSFILTER = 0x29 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIP_MULTICAST_ALL = 0x31 + + sysICMP_FILTER = 0x1 + + sysSO_EE_ORIGIN_NONE = 0x0 + sysSO_EE_ORIGIN_LOCAL = 0x1 + sysSO_EE_ORIGIN_ICMP = 0x2 + sysSO_EE_ORIGIN_ICMP6 = 0x3 + sysSO_EE_ORIGIN_TXSTATUS = 0x4 + sysSO_EE_ORIGIN_TIMESTAMPING = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + sizeofSockExtendedErr = 0x10 + + sizeofIPMreq = 0x8 + sizeofIPMreqn = 0xc + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPFilter = 0x4 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + X__pad [8]uint8 +} + +type inetPktinfo struct { + Ifindex int32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type sockExtendedErr struct { + Errno uint32 + Origin uint8 + Type uint8 + Code uint8 + Pad uint8 + Info uint32 + Data uint32 +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqn struct { + Multiaddr [4]byte /* in_addr */ + Address [4]byte /* in_addr */ + Ifindex int32 +} + +type ipMreqSource struct { + Multiaddr uint32 + Interface uint32 + Sourceaddr uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpFilter struct { + Data uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_netbsd.go b/vendor/golang.org/x/net/ipv4/zsys_netbsd.go new file mode 100644 index 00000000..fd3624d9 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_netbsd.go @@ -0,0 +1,30 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_netbsd.go + +package ipv4 + +const ( + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_RECVOPTS = 0x5 + sysIP_RECVRETOPTS = 0x6 + sysIP_RECVDSTADDR = 0x7 + sysIP_RETOPTS = 0x8 + sysIP_RECVIF = 0x14 + sysIP_RECVTTL = 0x17 + + sysIP_MULTICAST_IF = 0x9 + sysIP_MULTICAST_TTL = 0xa + sysIP_MULTICAST_LOOP = 0xb + sysIP_ADD_MEMBERSHIP = 0xc + sysIP_DROP_MEMBERSHIP = 0xd + + sizeofIPMreq = 0x8 +) + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_openbsd.go b/vendor/golang.org/x/net/ipv4/zsys_openbsd.go new file mode 100644 index 00000000..12f36be7 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_openbsd.go @@ -0,0 +1,30 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_openbsd.go + +package ipv4 + +const ( + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_RECVOPTS = 0x5 + sysIP_RECVRETOPTS = 0x6 + sysIP_RECVDSTADDR = 0x7 + sysIP_RETOPTS = 0x8 + sysIP_RECVIF = 0x1e + sysIP_RECVTTL = 0x1f + + sysIP_MULTICAST_IF = 0x9 + sysIP_MULTICAST_TTL = 0xa + sysIP_MULTICAST_LOOP = 0xb + sysIP_ADD_MEMBERSHIP = 0xc + sysIP_DROP_MEMBERSHIP = 0xd + + sizeofIPMreq = 0x8 +) + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} diff --git a/vendor/golang.org/x/net/ipv4/zsys_solaris.go b/vendor/golang.org/x/net/ipv4/zsys_solaris.go new file mode 100644 index 00000000..0a3875cc --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/zsys_solaris.go @@ -0,0 +1,100 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_solaris.go + +package ipv4 + +const ( + sysIP_OPTIONS = 0x1 + sysIP_HDRINCL = 0x2 + sysIP_TOS = 0x3 + sysIP_TTL = 0x4 + sysIP_RECVOPTS = 0x5 + sysIP_RECVRETOPTS = 0x6 + sysIP_RECVDSTADDR = 0x7 + sysIP_RETOPTS = 0x8 + sysIP_RECVIF = 0x9 + sysIP_RECVSLLA = 0xa + sysIP_RECVTTL = 0xb + + sysIP_MULTICAST_IF = 0x10 + sysIP_MULTICAST_TTL = 0x11 + sysIP_MULTICAST_LOOP = 0x12 + sysIP_ADD_MEMBERSHIP = 0x13 + sysIP_DROP_MEMBERSHIP = 0x14 + sysIP_BLOCK_SOURCE = 0x15 + sysIP_UNBLOCK_SOURCE = 0x16 + sysIP_ADD_SOURCE_MEMBERSHIP = 0x17 + sysIP_DROP_SOURCE_MEMBERSHIP = 0x18 + sysIP_NEXTHOP = 0x19 + + sysIP_PKTINFO = 0x1a + sysIP_RECVPKTINFO = 0x1a + sysIP_DONTFRAG = 0x1b + + sysIP_BOUND_IF = 0x41 + sysIP_UNSPEC_SRC = 0x42 + sysIP_BROADCAST_TTL = 0x43 + sysIP_DHCPINIT_IF = 0x45 + + sysIP_REUSEADDR = 0x104 + sysIP_DONTROUTE = 0x105 + sysIP_BROADCAST = 0x106 + + sysMCAST_JOIN_GROUP = 0x29 + sysMCAST_LEAVE_GROUP = 0x2a + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_JOIN_SOURCE_GROUP = 0x2d + sysMCAST_LEAVE_SOURCE_GROUP = 0x2e + + sizeofSockaddrStorage = 0x100 + sizeofSockaddrInet = 0x10 + sizeofInetPktinfo = 0xc + + sizeofIPMreq = 0x8 + sizeofIPMreqSource = 0xc + sizeofGroupReq = 0x104 + sizeofGroupSourceReq = 0x204 +) + +type sockaddrStorage struct { + Family uint16 + X_ss_pad1 [6]int8 + X_ss_align float64 + X_ss_pad2 [240]int8 +} + +type sockaddrInet struct { + Family uint16 + Port uint16 + Addr [4]byte /* in_addr */ + Zero [8]int8 +} + +type inetPktinfo struct { + Ifindex uint32 + Spec_dst [4]byte /* in_addr */ + Addr [4]byte /* in_addr */ +} + +type ipMreq struct { + Multiaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type ipMreqSource struct { + Multiaddr [4]byte /* in_addr */ + Sourceaddr [4]byte /* in_addr */ + Interface [4]byte /* in_addr */ +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [256]byte +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [256]byte + Pad_cgo_1 [256]byte +} diff --git a/vendor/golang.org/x/net/ipv6/batch.go b/vendor/golang.org/x/net/ipv6/batch.go new file mode 100644 index 00000000..4f5fe683 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/batch.go @@ -0,0 +1,119 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 + +package ipv6 + +import ( + "net" + "runtime" + "syscall" + + "golang.org/x/net/internal/socket" +) + +// BUG(mikio): On Windows, the ReadBatch and WriteBatch methods of +// PacketConn are not implemented. + +// A Message represents an IO message. +// +// type Message struct { +// Buffers [][]byte +// OOB []byte +// Addr net.Addr +// N int +// NN int +// Flags int +// } +// +// The Buffers fields represents a list of contiguous buffers, which +// can be used for vectored IO, for example, putting a header and a +// payload in each slice. +// When writing, the Buffers field must contain at least one byte to +// write. +// When reading, the Buffers field will always contain a byte to read. +// +// The OOB field contains protocol-specific control or miscellaneous +// ancillary data known as out-of-band data. +// It can be nil when not required. +// +// The Addr field specifies a destination address when writing. +// It can be nil when the underlying protocol of the endpoint uses +// connection-oriented communication. +// After a successful read, it may contain the source address on the +// received packet. +// +// The N field indicates the number of bytes read or written from/to +// Buffers. +// +// The NN field indicates the number of bytes read or written from/to +// OOB. +// +// The Flags field contains protocol-specific information on the +// received message. +type Message = socket.Message + +// ReadBatch reads a batch of messages. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_PEEK. +// +// On a successful read it returns the number of messages received, up +// to len(ms). +// +// On Linux, a batch read will be optimized. +// On other platforms, this method will read only a single message. +func (c *payloadHandler) ReadBatch(ms []Message, flags int) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + switch runtime.GOOS { + case "linux": + n, err := c.RecvMsgs([]socket.Message(ms), flags) + if err != nil { + err = &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + return n, err + default: + n := 1 + err := c.RecvMsg(&ms[0], flags) + if err != nil { + n = 0 + err = &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + return n, err + } +} + +// WriteBatch writes a batch of messages. +// +// The provided flags is a set of platform-dependent flags, such as +// syscall.MSG_DONTROUTE. +// +// It returns the number of messages written on a successful write. +// +// On Linux, a batch write will be optimized. +// On other platforms, this method will write only a single message. +func (c *payloadHandler) WriteBatch(ms []Message, flags int) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + switch runtime.GOOS { + case "linux": + n, err := c.SendMsgs([]socket.Message(ms), flags) + if err != nil { + err = &net.OpError{Op: "write", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + return n, err + default: + n := 1 + err := c.SendMsg(&ms[0], flags) + if err != nil { + n = 0 + err = &net.OpError{Op: "write", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + return n, err + } +} diff --git a/vendor/golang.org/x/net/ipv6/control.go b/vendor/golang.org/x/net/ipv6/control.go new file mode 100644 index 00000000..2da64441 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/control.go @@ -0,0 +1,187 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "fmt" + "net" + "sync" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +// Note that RFC 3542 obsoletes RFC 2292 but OS X Snow Leopard and the +// former still support RFC 2292 only. Please be aware that almost +// all protocol implementations prohibit using a combination of RFC +// 2292 and RFC 3542 for some practical reasons. + +type rawOpt struct { + sync.RWMutex + cflags ControlFlags +} + +func (c *rawOpt) set(f ControlFlags) { c.cflags |= f } +func (c *rawOpt) clear(f ControlFlags) { c.cflags &^= f } +func (c *rawOpt) isset(f ControlFlags) bool { return c.cflags&f != 0 } + +// A ControlFlags represents per packet basis IP-level socket option +// control flags. +type ControlFlags uint + +const ( + FlagTrafficClass ControlFlags = 1 << iota // pass the traffic class on the received packet + FlagHopLimit // pass the hop limit on the received packet + FlagSrc // pass the source address on the received packet + FlagDst // pass the destination address on the received packet + FlagInterface // pass the interface index on the received packet + FlagPathMTU // pass the path MTU on the received packet path +) + +const flagPacketInfo = FlagDst | FlagInterface + +// A ControlMessage represents per packet basis IP-level socket +// options. +type ControlMessage struct { + // Receiving socket options: SetControlMessage allows to + // receive the options from the protocol stack using ReadFrom + // method of PacketConn. + // + // Specifying socket options: ControlMessage for WriteTo + // method of PacketConn allows to send the options to the + // protocol stack. + // + TrafficClass int // traffic class, must be 1 <= value <= 255 when specifying + HopLimit int // hop limit, must be 1 <= value <= 255 when specifying + Src net.IP // source address, specifying only + Dst net.IP // destination address, receiving only + IfIndex int // interface index, must be 1 <= value when specifying + NextHop net.IP // next hop address, specifying only + MTU int // path MTU, receiving only +} + +func (cm *ControlMessage) String() string { + if cm == nil { + return "" + } + return fmt.Sprintf("tclass=%#x hoplim=%d src=%v dst=%v ifindex=%d nexthop=%v mtu=%d", cm.TrafficClass, cm.HopLimit, cm.Src, cm.Dst, cm.IfIndex, cm.NextHop, cm.MTU) +} + +// Marshal returns the binary encoding of cm. +func (cm *ControlMessage) Marshal() []byte { + if cm == nil { + return nil + } + var l int + tclass := false + if ctlOpts[ctlTrafficClass].name > 0 && cm.TrafficClass > 0 { + tclass = true + l += socket.ControlMessageSpace(ctlOpts[ctlTrafficClass].length) + } + hoplimit := false + if ctlOpts[ctlHopLimit].name > 0 && cm.HopLimit > 0 { + hoplimit = true + l += socket.ControlMessageSpace(ctlOpts[ctlHopLimit].length) + } + pktinfo := false + if ctlOpts[ctlPacketInfo].name > 0 && (cm.Src.To16() != nil && cm.Src.To4() == nil || cm.IfIndex > 0) { + pktinfo = true + l += socket.ControlMessageSpace(ctlOpts[ctlPacketInfo].length) + } + nexthop := false + if ctlOpts[ctlNextHop].name > 0 && cm.NextHop.To16() != nil && cm.NextHop.To4() == nil { + nexthop = true + l += socket.ControlMessageSpace(ctlOpts[ctlNextHop].length) + } + var b []byte + if l > 0 { + b = make([]byte, l) + bb := b + if tclass { + bb = ctlOpts[ctlTrafficClass].marshal(bb, cm) + } + if hoplimit { + bb = ctlOpts[ctlHopLimit].marshal(bb, cm) + } + if pktinfo { + bb = ctlOpts[ctlPacketInfo].marshal(bb, cm) + } + if nexthop { + bb = ctlOpts[ctlNextHop].marshal(bb, cm) + } + } + return b +} + +// Parse parses b as a control message and stores the result in cm. +func (cm *ControlMessage) Parse(b []byte) error { + ms, err := socket.ControlMessage(b).Parse() + if err != nil { + return err + } + for _, m := range ms { + lvl, typ, l, err := m.ParseHeader() + if err != nil { + return err + } + if lvl != iana.ProtocolIPv6 { + continue + } + switch { + case typ == ctlOpts[ctlTrafficClass].name && l >= ctlOpts[ctlTrafficClass].length: + ctlOpts[ctlTrafficClass].parse(cm, m.Data(l)) + case typ == ctlOpts[ctlHopLimit].name && l >= ctlOpts[ctlHopLimit].length: + ctlOpts[ctlHopLimit].parse(cm, m.Data(l)) + case typ == ctlOpts[ctlPacketInfo].name && l >= ctlOpts[ctlPacketInfo].length: + ctlOpts[ctlPacketInfo].parse(cm, m.Data(l)) + case typ == ctlOpts[ctlPathMTU].name && l >= ctlOpts[ctlPathMTU].length: + ctlOpts[ctlPathMTU].parse(cm, m.Data(l)) + } + } + return nil +} + +// NewControlMessage returns a new control message. +// +// The returned message is large enough for options specified by cf. +func NewControlMessage(cf ControlFlags) []byte { + opt := rawOpt{cflags: cf} + var l int + if opt.isset(FlagTrafficClass) && ctlOpts[ctlTrafficClass].name > 0 { + l += socket.ControlMessageSpace(ctlOpts[ctlTrafficClass].length) + } + if opt.isset(FlagHopLimit) && ctlOpts[ctlHopLimit].name > 0 { + l += socket.ControlMessageSpace(ctlOpts[ctlHopLimit].length) + } + if opt.isset(flagPacketInfo) && ctlOpts[ctlPacketInfo].name > 0 { + l += socket.ControlMessageSpace(ctlOpts[ctlPacketInfo].length) + } + if opt.isset(FlagPathMTU) && ctlOpts[ctlPathMTU].name > 0 { + l += socket.ControlMessageSpace(ctlOpts[ctlPathMTU].length) + } + var b []byte + if l > 0 { + b = make([]byte, l) + } + return b +} + +// Ancillary data socket options +const ( + ctlTrafficClass = iota // header field + ctlHopLimit // header field + ctlPacketInfo // inbound or outbound packet path + ctlNextHop // nexthop + ctlPathMTU // path mtu + ctlMax +) + +// A ctlOpt represents a binding for ancillary data socket option. +type ctlOpt struct { + name int // option name, must be equal or greater than 1 + length int // option length + marshal func([]byte, *ControlMessage) []byte + parse func(*ControlMessage, []byte) +} diff --git a/vendor/golang.org/x/net/ipv6/control_rfc2292_unix.go b/vendor/golang.org/x/net/ipv6/control_rfc2292_unix.go new file mode 100644 index 00000000..9fd9eb15 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/control_rfc2292_unix.go @@ -0,0 +1,48 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package ipv6 + +import ( + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +func marshal2292HopLimit(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIPv6, sysIPV6_2292HOPLIMIT, 4) + if cm != nil { + socket.NativeEndian.PutUint32(m.Data(4), uint32(cm.HopLimit)) + } + return m.Next(4) +} + +func marshal2292PacketInfo(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIPv6, sysIPV6_2292PKTINFO, sizeofInet6Pktinfo) + if cm != nil { + pi := (*inet6Pktinfo)(unsafe.Pointer(&m.Data(sizeofInet6Pktinfo)[0])) + if ip := cm.Src.To16(); ip != nil && ip.To4() == nil { + copy(pi.Addr[:], ip) + } + if cm.IfIndex > 0 { + pi.setIfindex(cm.IfIndex) + } + } + return m.Next(sizeofInet6Pktinfo) +} + +func marshal2292NextHop(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIPv6, sysIPV6_2292NEXTHOP, sizeofSockaddrInet6) + if cm != nil { + sa := (*sockaddrInet6)(unsafe.Pointer(&m.Data(sizeofSockaddrInet6)[0])) + sa.setSockaddr(cm.NextHop, cm.IfIndex) + } + return m.Next(sizeofSockaddrInet6) +} diff --git a/vendor/golang.org/x/net/ipv6/control_rfc3542_unix.go b/vendor/golang.org/x/net/ipv6/control_rfc3542_unix.go new file mode 100644 index 00000000..eec529c2 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/control_rfc3542_unix.go @@ -0,0 +1,94 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package ipv6 + +import ( + "net" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +func marshalTrafficClass(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIPv6, sysIPV6_TCLASS, 4) + if cm != nil { + socket.NativeEndian.PutUint32(m.Data(4), uint32(cm.TrafficClass)) + } + return m.Next(4) +} + +func parseTrafficClass(cm *ControlMessage, b []byte) { + cm.TrafficClass = int(socket.NativeEndian.Uint32(b[:4])) +} + +func marshalHopLimit(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIPv6, sysIPV6_HOPLIMIT, 4) + if cm != nil { + socket.NativeEndian.PutUint32(m.Data(4), uint32(cm.HopLimit)) + } + return m.Next(4) +} + +func parseHopLimit(cm *ControlMessage, b []byte) { + cm.HopLimit = int(socket.NativeEndian.Uint32(b[:4])) +} + +func marshalPacketInfo(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIPv6, sysIPV6_PKTINFO, sizeofInet6Pktinfo) + if cm != nil { + pi := (*inet6Pktinfo)(unsafe.Pointer(&m.Data(sizeofInet6Pktinfo)[0])) + if ip := cm.Src.To16(); ip != nil && ip.To4() == nil { + copy(pi.Addr[:], ip) + } + if cm.IfIndex > 0 { + pi.setIfindex(cm.IfIndex) + } + } + return m.Next(sizeofInet6Pktinfo) +} + +func parsePacketInfo(cm *ControlMessage, b []byte) { + pi := (*inet6Pktinfo)(unsafe.Pointer(&b[0])) + if len(cm.Dst) < net.IPv6len { + cm.Dst = make(net.IP, net.IPv6len) + } + copy(cm.Dst, pi.Addr[:]) + cm.IfIndex = int(pi.Ifindex) +} + +func marshalNextHop(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIPv6, sysIPV6_NEXTHOP, sizeofSockaddrInet6) + if cm != nil { + sa := (*sockaddrInet6)(unsafe.Pointer(&m.Data(sizeofSockaddrInet6)[0])) + sa.setSockaddr(cm.NextHop, cm.IfIndex) + } + return m.Next(sizeofSockaddrInet6) +} + +func parseNextHop(cm *ControlMessage, b []byte) { +} + +func marshalPathMTU(b []byte, cm *ControlMessage) []byte { + m := socket.ControlMessage(b) + m.MarshalHeader(iana.ProtocolIPv6, sysIPV6_PATHMTU, sizeofIPv6Mtuinfo) + return m.Next(sizeofIPv6Mtuinfo) +} + +func parsePathMTU(cm *ControlMessage, b []byte) { + mi := (*ipv6Mtuinfo)(unsafe.Pointer(&b[0])) + if len(cm.Dst) < net.IPv6len { + cm.Dst = make(net.IP, net.IPv6len) + } + copy(cm.Dst, mi.Addr.Addr[:]) + cm.IfIndex = int(mi.Addr.Scope_id) + cm.MTU = int(mi.Mtu) +} diff --git a/vendor/golang.org/x/net/ipv6/control_stub.go b/vendor/golang.org/x/net/ipv6/control_stub.go new file mode 100644 index 00000000..a045f28f --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/control_stub.go @@ -0,0 +1,13 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package ipv6 + +import "golang.org/x/net/internal/socket" + +func setControlMessage(c *socket.Conn, opt *rawOpt, cf ControlFlags, on bool) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv6/control_unix.go b/vendor/golang.org/x/net/ipv6/control_unix.go new file mode 100644 index 00000000..66515060 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/control_unix.go @@ -0,0 +1,55 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package ipv6 + +import "golang.org/x/net/internal/socket" + +func setControlMessage(c *socket.Conn, opt *rawOpt, cf ControlFlags, on bool) error { + opt.Lock() + defer opt.Unlock() + if so, ok := sockOpts[ssoReceiveTrafficClass]; ok && cf&FlagTrafficClass != 0 { + if err := so.SetInt(c, boolint(on)); err != nil { + return err + } + if on { + opt.set(FlagTrafficClass) + } else { + opt.clear(FlagTrafficClass) + } + } + if so, ok := sockOpts[ssoReceiveHopLimit]; ok && cf&FlagHopLimit != 0 { + if err := so.SetInt(c, boolint(on)); err != nil { + return err + } + if on { + opt.set(FlagHopLimit) + } else { + opt.clear(FlagHopLimit) + } + } + if so, ok := sockOpts[ssoReceivePacketInfo]; ok && cf&flagPacketInfo != 0 { + if err := so.SetInt(c, boolint(on)); err != nil { + return err + } + if on { + opt.set(cf & flagPacketInfo) + } else { + opt.clear(cf & flagPacketInfo) + } + } + if so, ok := sockOpts[ssoReceivePathMTU]; ok && cf&FlagPathMTU != 0 { + if err := so.SetInt(c, boolint(on)); err != nil { + return err + } + if on { + opt.set(FlagPathMTU) + } else { + opt.clear(FlagPathMTU) + } + } + return nil +} diff --git a/vendor/golang.org/x/net/ipv6/control_windows.go b/vendor/golang.org/x/net/ipv6/control_windows.go new file mode 100644 index 00000000..ef2563b3 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/control_windows.go @@ -0,0 +1,16 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "syscall" + + "golang.org/x/net/internal/socket" +) + +func setControlMessage(c *socket.Conn, opt *rawOpt, cf ControlFlags, on bool) error { + // TODO(mikio): implement this + return syscall.EWINDOWS +} diff --git a/vendor/golang.org/x/net/ipv6/defs_darwin.go b/vendor/golang.org/x/net/ipv6/defs_darwin.go new file mode 100644 index 00000000..55ddc116 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_darwin.go @@ -0,0 +1,112 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#define __APPLE_USE_RFC_3542 +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + sysIPV6_2292PKTINFO = C.IPV6_2292PKTINFO + sysIPV6_2292HOPLIMIT = C.IPV6_2292HOPLIMIT + sysIPV6_2292NEXTHOP = C.IPV6_2292NEXTHOP + sysIPV6_2292HOPOPTS = C.IPV6_2292HOPOPTS + sysIPV6_2292DSTOPTS = C.IPV6_2292DSTOPTS + sysIPV6_2292RTHDR = C.IPV6_2292RTHDR + + sysIPV6_2292PKTOPTIONS = C.IPV6_2292PKTOPTIONS + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + sysIPV6_TCLASS = C.IPV6_TCLASS + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_AUTOFLOWLABEL = C.IPV6_AUTOFLOWLABEL + + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_PREFER_TEMPADDR = C.IPV6_PREFER_TEMPADDR + + sysIPV6_MSFILTER = C.IPV6_MSFILTER + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + + sysIPV6_BOUND_IF = C.IPV6_BOUND_IF + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv6/defs_dragonfly.go b/vendor/golang.org/x/net/ipv6/defs_dragonfly.go new file mode 100644 index 00000000..a4c383a5 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_dragonfly.go @@ -0,0 +1,84 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_AUTOFLOWLABEL = C.IPV6_AUTOFLOWLABEL + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_PREFER_TEMPADDR = C.IPV6_PREFER_TEMPADDR + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_freebsd.go b/vendor/golang.org/x/net/ipv6/defs_freebsd.go new file mode 100644 index 00000000..53e62538 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_freebsd.go @@ -0,0 +1,105 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_AUTOFLOWLABEL = C.IPV6_AUTOFLOWLABEL + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_PREFER_TEMPADDR = C.IPV6_PREFER_TEMPADDR + + sysIPV6_BINDANY = C.IPV6_BINDANY + + sysIPV6_MSFILTER = C.IPV6_MSFILTER + + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_linux.go b/vendor/golang.org/x/net/ipv6/defs_linux.go new file mode 100644 index 00000000..3308cb2c --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_linux.go @@ -0,0 +1,147 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include +#include +#include +#include +#include +*/ +import "C" + +const ( + sysIPV6_ADDRFORM = C.IPV6_ADDRFORM + sysIPV6_2292PKTINFO = C.IPV6_2292PKTINFO + sysIPV6_2292HOPOPTS = C.IPV6_2292HOPOPTS + sysIPV6_2292DSTOPTS = C.IPV6_2292DSTOPTS + sysIPV6_2292RTHDR = C.IPV6_2292RTHDR + sysIPV6_2292PKTOPTIONS = C.IPV6_2292PKTOPTIONS + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_2292HOPLIMIT = C.IPV6_2292HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_FLOWINFO = C.IPV6_FLOWINFO + + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_ADD_MEMBERSHIP = C.IPV6_ADD_MEMBERSHIP + sysIPV6_DROP_MEMBERSHIP = C.IPV6_DROP_MEMBERSHIP + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + sysMCAST_MSFILTER = C.MCAST_MSFILTER + sysIPV6_ROUTER_ALERT = C.IPV6_ROUTER_ALERT + sysIPV6_MTU_DISCOVER = C.IPV6_MTU_DISCOVER + sysIPV6_MTU = C.IPV6_MTU + sysIPV6_RECVERR = C.IPV6_RECVERR + sysIPV6_V6ONLY = C.IPV6_V6ONLY + sysIPV6_JOIN_ANYCAST = C.IPV6_JOIN_ANYCAST + sysIPV6_LEAVE_ANYCAST = C.IPV6_LEAVE_ANYCAST + + //sysIPV6_PMTUDISC_DONT = C.IPV6_PMTUDISC_DONT + //sysIPV6_PMTUDISC_WANT = C.IPV6_PMTUDISC_WANT + //sysIPV6_PMTUDISC_DO = C.IPV6_PMTUDISC_DO + //sysIPV6_PMTUDISC_PROBE = C.IPV6_PMTUDISC_PROBE + //sysIPV6_PMTUDISC_INTERFACE = C.IPV6_PMTUDISC_INTERFACE + //sysIPV6_PMTUDISC_OMIT = C.IPV6_PMTUDISC_OMIT + + sysIPV6_FLOWLABEL_MGR = C.IPV6_FLOWLABEL_MGR + sysIPV6_FLOWINFO_SEND = C.IPV6_FLOWINFO_SEND + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + sysIPV6_XFRM_POLICY = C.IPV6_XFRM_POLICY + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RTHDR = C.IPV6_RTHDR + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + sysIPV6_PATHMTU = C.IPV6_PATHMTU + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + sysIPV6_TCLASS = C.IPV6_TCLASS + + sysIPV6_ADDR_PREFERENCES = C.IPV6_ADDR_PREFERENCES + + sysIPV6_PREFER_SRC_TMP = C.IPV6_PREFER_SRC_TMP + sysIPV6_PREFER_SRC_PUBLIC = C.IPV6_PREFER_SRC_PUBLIC + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = C.IPV6_PREFER_SRC_PUBTMP_DEFAULT + sysIPV6_PREFER_SRC_COA = C.IPV6_PREFER_SRC_COA + sysIPV6_PREFER_SRC_HOME = C.IPV6_PREFER_SRC_HOME + sysIPV6_PREFER_SRC_CGA = C.IPV6_PREFER_SRC_CGA + sysIPV6_PREFER_SRC_NONCGA = C.IPV6_PREFER_SRC_NONCGA + + sysIPV6_MINHOPCOUNT = C.IPV6_MINHOPCOUNT + + sysIPV6_ORIGDSTADDR = C.IPV6_ORIGDSTADDR + sysIPV6_RECVORIGDSTADDR = C.IPV6_RECVORIGDSTADDR + sysIPV6_TRANSPARENT = C.IPV6_TRANSPARENT + sysIPV6_UNICAST_IF = C.IPV6_UNICAST_IF + + sysICMPV6_FILTER = C.ICMPV6_FILTER + + sysICMPV6_FILTER_BLOCK = C.ICMPV6_FILTER_BLOCK + sysICMPV6_FILTER_PASS = C.ICMPV6_FILTER_PASS + sysICMPV6_FILTER_BLOCKOTHERS = C.ICMPV6_FILTER_BLOCKOTHERS + sysICMPV6_FILTER_PASSONLY = C.ICMPV6_FILTER_PASSONLY + + sysSOL_SOCKET = C.SOL_SOCKET + sysSO_ATTACH_FILTER = C.SO_ATTACH_FILTER + + sizeofKernelSockaddrStorage = C.sizeof_struct___kernel_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + sizeofIPv6FlowlabelReq = C.sizeof_struct_in6_flowlabel_req + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter + + sizeofSockFprog = C.sizeof_struct_sock_fprog +) + +type kernelSockaddrStorage C.struct___kernel_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6FlowlabelReq C.struct_in6_flowlabel_req + +type ipv6Mreq C.struct_ipv6_mreq + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req + +type icmpv6Filter C.struct_icmp6_filter + +type sockFProg C.struct_sock_fprog + +type sockFilter C.struct_sock_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_netbsd.go b/vendor/golang.org/x/net/ipv6/defs_netbsd.go new file mode 100644 index 00000000..be9ceb9c --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_netbsd.go @@ -0,0 +1,80 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_openbsd.go b/vendor/golang.org/x/net/ipv6/defs_openbsd.go new file mode 100644 index 00000000..177ddf87 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_openbsd.go @@ -0,0 +1,89 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_AUTH_LEVEL = C.IPV6_AUTH_LEVEL + sysIPV6_ESP_TRANS_LEVEL = C.IPV6_ESP_TRANS_LEVEL + sysIPV6_ESP_NETWORK_LEVEL = C.IPV6_ESP_NETWORK_LEVEL + sysIPSEC6_OUTSA = C.IPSEC6_OUTSA + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_AUTOFLOWLABEL = C.IPV6_AUTOFLOWLABEL + sysIPV6_IPCOMP_LEVEL = C.IPV6_IPCOMP_LEVEL + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + sysIPV6_PIPEX = C.IPV6_PIPEX + + sysIPV6_RTABLE = C.IPV6_RTABLE + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_solaris.go b/vendor/golang.org/x/net/ipv6/defs_solaris.go new file mode 100644 index 00000000..0f8ce2b4 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_solaris.go @@ -0,0 +1,114 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + + sysIPV6_RTHDR = C.IPV6_RTHDR + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + + sysIPV6_RECVRTHDRDSTOPTS = C.IPV6_RECVRTHDRDSTOPTS + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + sysIPV6_SEC_OPT = C.IPV6_SEC_OPT + sysIPV6_SRC_PREFERENCES = C.IPV6_SRC_PREFERENCES + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + sysIPV6_PATHMTU = C.IPV6_PATHMTU + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + + sysIPV6_PREFER_SRC_HOME = C.IPV6_PREFER_SRC_HOME + sysIPV6_PREFER_SRC_COA = C.IPV6_PREFER_SRC_COA + sysIPV6_PREFER_SRC_PUBLIC = C.IPV6_PREFER_SRC_PUBLIC + sysIPV6_PREFER_SRC_TMP = C.IPV6_PREFER_SRC_TMP + sysIPV6_PREFER_SRC_NONCGA = C.IPV6_PREFER_SRC_NONCGA + sysIPV6_PREFER_SRC_CGA = C.IPV6_PREFER_SRC_CGA + + sysIPV6_PREFER_SRC_MIPMASK = C.IPV6_PREFER_SRC_MIPMASK + sysIPV6_PREFER_SRC_MIPDEFAULT = C.IPV6_PREFER_SRC_MIPDEFAULT + sysIPV6_PREFER_SRC_TMPMASK = C.IPV6_PREFER_SRC_TMPMASK + sysIPV6_PREFER_SRC_TMPDEFAULT = C.IPV6_PREFER_SRC_TMPDEFAULT + sysIPV6_PREFER_SRC_CGAMASK = C.IPV6_PREFER_SRC_CGAMASK + sysIPV6_PREFER_SRC_CGADEFAULT = C.IPV6_PREFER_SRC_CGADEFAULT + + sysIPV6_PREFER_SRC_MASK = C.IPV6_PREFER_SRC_MASK + + sysIPV6_PREFER_SRC_DEFAULT = C.IPV6_PREFER_SRC_DEFAULT + + sysIPV6_BOUND_IF = C.IPV6_BOUND_IF + sysIPV6_UNSPEC_SRC = C.IPV6_UNSPEC_SRC + + sysICMP6_FILTER = C.ICMP6_FILTER + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/dgramopt.go b/vendor/golang.org/x/net/ipv6/dgramopt.go new file mode 100644 index 00000000..703dafe8 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/dgramopt.go @@ -0,0 +1,302 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "syscall" + + "golang.org/x/net/bpf" +) + +// MulticastHopLimit returns the hop limit field value for outgoing +// multicast packets. +func (c *dgramOpt) MulticastHopLimit() (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastHopLimit] + if !ok { + return 0, errOpNoSupport + } + return so.GetInt(c.Conn) +} + +// SetMulticastHopLimit sets the hop limit field value for future +// outgoing multicast packets. +func (c *dgramOpt) SetMulticastHopLimit(hoplim int) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastHopLimit] + if !ok { + return errOpNoSupport + } + return so.SetInt(c.Conn, hoplim) +} + +// MulticastInterface returns the default interface for multicast +// packet transmissions. +func (c *dgramOpt) MulticastInterface() (*net.Interface, error) { + if !c.ok() { + return nil, syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastInterface] + if !ok { + return nil, errOpNoSupport + } + return so.getMulticastInterface(c.Conn) +} + +// SetMulticastInterface sets the default interface for future +// multicast packet transmissions. +func (c *dgramOpt) SetMulticastInterface(ifi *net.Interface) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastInterface] + if !ok { + return errOpNoSupport + } + return so.setMulticastInterface(c.Conn, ifi) +} + +// MulticastLoopback reports whether transmitted multicast packets +// should be copied and send back to the originator. +func (c *dgramOpt) MulticastLoopback() (bool, error) { + if !c.ok() { + return false, syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastLoopback] + if !ok { + return false, errOpNoSupport + } + on, err := so.GetInt(c.Conn) + if err != nil { + return false, err + } + return on == 1, nil +} + +// SetMulticastLoopback sets whether transmitted multicast packets +// should be copied and send back to the originator. +func (c *dgramOpt) SetMulticastLoopback(on bool) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoMulticastLoopback] + if !ok { + return errOpNoSupport + } + return so.SetInt(c.Conn, boolint(on)) +} + +// JoinGroup joins the group address group on the interface ifi. +// By default all sources that can cast data to group are accepted. +// It's possible to mute and unmute data transmission from a specific +// source by using ExcludeSourceSpecificGroup and +// IncludeSourceSpecificGroup. +// JoinGroup uses the system assigned multicast interface when ifi is +// nil, although this is not recommended because the assignment +// depends on platforms and sometimes it might require routing +// configuration. +func (c *dgramOpt) JoinGroup(ifi *net.Interface, group net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoJoinGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP16(group) + if grp == nil { + return errMissingAddress + } + return so.setGroup(c.Conn, ifi, grp) +} + +// LeaveGroup leaves the group address group on the interface ifi +// regardless of whether the group is any-source group or +// source-specific group. +func (c *dgramOpt) LeaveGroup(ifi *net.Interface, group net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoLeaveGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP16(group) + if grp == nil { + return errMissingAddress + } + return so.setGroup(c.Conn, ifi, grp) +} + +// JoinSourceSpecificGroup joins the source-specific group comprising +// group and source on the interface ifi. +// JoinSourceSpecificGroup uses the system assigned multicast +// interface when ifi is nil, although this is not recommended because +// the assignment depends on platforms and sometimes it might require +// routing configuration. +func (c *dgramOpt) JoinSourceSpecificGroup(ifi *net.Interface, group, source net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoJoinSourceGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP16(group) + if grp == nil { + return errMissingAddress + } + src := netAddrToIP16(source) + if src == nil { + return errMissingAddress + } + return so.setSourceGroup(c.Conn, ifi, grp, src) +} + +// LeaveSourceSpecificGroup leaves the source-specific group on the +// interface ifi. +func (c *dgramOpt) LeaveSourceSpecificGroup(ifi *net.Interface, group, source net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoLeaveSourceGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP16(group) + if grp == nil { + return errMissingAddress + } + src := netAddrToIP16(source) + if src == nil { + return errMissingAddress + } + return so.setSourceGroup(c.Conn, ifi, grp, src) +} + +// ExcludeSourceSpecificGroup excludes the source-specific group from +// the already joined any-source groups by JoinGroup on the interface +// ifi. +func (c *dgramOpt) ExcludeSourceSpecificGroup(ifi *net.Interface, group, source net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoBlockSourceGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP16(group) + if grp == nil { + return errMissingAddress + } + src := netAddrToIP16(source) + if src == nil { + return errMissingAddress + } + return so.setSourceGroup(c.Conn, ifi, grp, src) +} + +// IncludeSourceSpecificGroup includes the excluded source-specific +// group by ExcludeSourceSpecificGroup again on the interface ifi. +func (c *dgramOpt) IncludeSourceSpecificGroup(ifi *net.Interface, group, source net.Addr) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoUnblockSourceGroup] + if !ok { + return errOpNoSupport + } + grp := netAddrToIP16(group) + if grp == nil { + return errMissingAddress + } + src := netAddrToIP16(source) + if src == nil { + return errMissingAddress + } + return so.setSourceGroup(c.Conn, ifi, grp, src) +} + +// Checksum reports whether the kernel will compute, store or verify a +// checksum for both incoming and outgoing packets. If on is true, it +// returns an offset in bytes into the data of where the checksum +// field is located. +func (c *dgramOpt) Checksum() (on bool, offset int, err error) { + if !c.ok() { + return false, 0, syscall.EINVAL + } + so, ok := sockOpts[ssoChecksum] + if !ok { + return false, 0, errOpNoSupport + } + offset, err = so.GetInt(c.Conn) + if err != nil { + return false, 0, err + } + if offset < 0 { + return false, 0, nil + } + return true, offset, nil +} + +// SetChecksum enables the kernel checksum processing. If on is ture, +// the offset should be an offset in bytes into the data of where the +// checksum field is located. +func (c *dgramOpt) SetChecksum(on bool, offset int) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoChecksum] + if !ok { + return errOpNoSupport + } + if !on { + offset = -1 + } + return so.SetInt(c.Conn, offset) +} + +// ICMPFilter returns an ICMP filter. +func (c *dgramOpt) ICMPFilter() (*ICMPFilter, error) { + if !c.ok() { + return nil, syscall.EINVAL + } + so, ok := sockOpts[ssoICMPFilter] + if !ok { + return nil, errOpNoSupport + } + return so.getICMPFilter(c.Conn) +} + +// SetICMPFilter deploys the ICMP filter. +func (c *dgramOpt) SetICMPFilter(f *ICMPFilter) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoICMPFilter] + if !ok { + return errOpNoSupport + } + return so.setICMPFilter(c.Conn, f) +} + +// SetBPF attaches a BPF program to the connection. +// +// Only supported on Linux. +func (c *dgramOpt) SetBPF(filter []bpf.RawInstruction) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoAttachFilter] + if !ok { + return errOpNoSupport + } + return so.setBPF(c.Conn, filter) +} diff --git a/vendor/golang.org/x/net/ipv6/doc.go b/vendor/golang.org/x/net/ipv6/doc.go new file mode 100644 index 00000000..664a97de --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/doc.go @@ -0,0 +1,243 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ipv6 implements IP-level socket options for the Internet +// Protocol version 6. +// +// The package provides IP-level socket options that allow +// manipulation of IPv6 facilities. +// +// The IPv6 protocol is defined in RFC 8200. +// Socket interface extensions are defined in RFC 3493, RFC 3542 and +// RFC 3678. +// MLDv1 and MLDv2 are defined in RFC 2710 and RFC 3810. +// Source-specific multicast is defined in RFC 4607. +// +// On Darwin, this package requires OS X Mavericks version 10.9 or +// above, or equivalent. +// +// +// Unicasting +// +// The options for unicasting are available for net.TCPConn, +// net.UDPConn and net.IPConn which are created as network connections +// that use the IPv6 transport. When a single TCP connection carrying +// a data flow of multiple packets needs to indicate the flow is +// important, Conn is used to set the traffic class field on the IPv6 +// header for each packet. +// +// ln, err := net.Listen("tcp6", "[::]:1024") +// if err != nil { +// // error handling +// } +// defer ln.Close() +// for { +// c, err := ln.Accept() +// if err != nil { +// // error handling +// } +// go func(c net.Conn) { +// defer c.Close() +// +// The outgoing packets will be labeled DiffServ assured forwarding +// class 1 low drop precedence, known as AF11 packets. +// +// if err := ipv6.NewConn(c).SetTrafficClass(0x28); err != nil { +// // error handling +// } +// if _, err := c.Write(data); err != nil { +// // error handling +// } +// }(c) +// } +// +// +// Multicasting +// +// The options for multicasting are available for net.UDPConn and +// net.IPconn which are created as network connections that use the +// IPv6 transport. A few network facilities must be prepared before +// you begin multicasting, at a minimum joining network interfaces and +// multicast groups. +// +// en0, err := net.InterfaceByName("en0") +// if err != nil { +// // error handling +// } +// en1, err := net.InterfaceByIndex(911) +// if err != nil { +// // error handling +// } +// group := net.ParseIP("ff02::114") +// +// First, an application listens to an appropriate address with an +// appropriate service port. +// +// c, err := net.ListenPacket("udp6", "[::]:1024") +// if err != nil { +// // error handling +// } +// defer c.Close() +// +// Second, the application joins multicast groups, starts listening to +// the groups on the specified network interfaces. Note that the +// service port for transport layer protocol does not matter with this +// operation as joining groups affects only network and link layer +// protocols, such as IPv6 and Ethernet. +// +// p := ipv6.NewPacketConn(c) +// if err := p.JoinGroup(en0, &net.UDPAddr{IP: group}); err != nil { +// // error handling +// } +// if err := p.JoinGroup(en1, &net.UDPAddr{IP: group}); err != nil { +// // error handling +// } +// +// The application might set per packet control message transmissions +// between the protocol stack within the kernel. When the application +// needs a destination address on an incoming packet, +// SetControlMessage of PacketConn is used to enable control message +// transmissions. +// +// if err := p.SetControlMessage(ipv6.FlagDst, true); err != nil { +// // error handling +// } +// +// The application could identify whether the received packets are +// of interest by using the control message that contains the +// destination address of the received packet. +// +// b := make([]byte, 1500) +// for { +// n, rcm, src, err := p.ReadFrom(b) +// if err != nil { +// // error handling +// } +// if rcm.Dst.IsMulticast() { +// if rcm.Dst.Equal(group) { +// // joined group, do something +// } else { +// // unknown group, discard +// continue +// } +// } +// +// The application can also send both unicast and multicast packets. +// +// p.SetTrafficClass(0x0) +// p.SetHopLimit(16) +// if _, err := p.WriteTo(data[:n], nil, src); err != nil { +// // error handling +// } +// dst := &net.UDPAddr{IP: group, Port: 1024} +// wcm := ipv6.ControlMessage{TrafficClass: 0xe0, HopLimit: 1} +// for _, ifi := range []*net.Interface{en0, en1} { +// wcm.IfIndex = ifi.Index +// if _, err := p.WriteTo(data[:n], &wcm, dst); err != nil { +// // error handling +// } +// } +// } +// +// +// More multicasting +// +// An application that uses PacketConn may join multiple multicast +// groups. For example, a UDP listener with port 1024 might join two +// different groups across over two different network interfaces by +// using: +// +// c, err := net.ListenPacket("udp6", "[::]:1024") +// if err != nil { +// // error handling +// } +// defer c.Close() +// p := ipv6.NewPacketConn(c) +// if err := p.JoinGroup(en0, &net.UDPAddr{IP: net.ParseIP("ff02::1:114")}); err != nil { +// // error handling +// } +// if err := p.JoinGroup(en0, &net.UDPAddr{IP: net.ParseIP("ff02::2:114")}); err != nil { +// // error handling +// } +// if err := p.JoinGroup(en1, &net.UDPAddr{IP: net.ParseIP("ff02::2:114")}); err != nil { +// // error handling +// } +// +// It is possible for multiple UDP listeners that listen on the same +// UDP port to join the same multicast group. The net package will +// provide a socket that listens to a wildcard address with reusable +// UDP port when an appropriate multicast address prefix is passed to +// the net.ListenPacket or net.ListenUDP. +// +// c1, err := net.ListenPacket("udp6", "[ff02::]:1024") +// if err != nil { +// // error handling +// } +// defer c1.Close() +// c2, err := net.ListenPacket("udp6", "[ff02::]:1024") +// if err != nil { +// // error handling +// } +// defer c2.Close() +// p1 := ipv6.NewPacketConn(c1) +// if err := p1.JoinGroup(en0, &net.UDPAddr{IP: net.ParseIP("ff02::114")}); err != nil { +// // error handling +// } +// p2 := ipv6.NewPacketConn(c2) +// if err := p2.JoinGroup(en0, &net.UDPAddr{IP: net.ParseIP("ff02::114")}); err != nil { +// // error handling +// } +// +// Also it is possible for the application to leave or rejoin a +// multicast group on the network interface. +// +// if err := p.LeaveGroup(en0, &net.UDPAddr{IP: net.ParseIP("ff02::114")}); err != nil { +// // error handling +// } +// if err := p.JoinGroup(en0, &net.UDPAddr{IP: net.ParseIP("ff01::114")}); err != nil { +// // error handling +// } +// +// +// Source-specific multicasting +// +// An application that uses PacketConn on MLDv2 supported platform is +// able to join source-specific multicast groups. +// The application may use JoinSourceSpecificGroup and +// LeaveSourceSpecificGroup for the operation known as "include" mode, +// +// ssmgroup := net.UDPAddr{IP: net.ParseIP("ff32::8000:9")} +// ssmsource := net.UDPAddr{IP: net.ParseIP("fe80::cafe")} +// if err := p.JoinSourceSpecificGroup(en0, &ssmgroup, &ssmsource); err != nil { +// // error handling +// } +// if err := p.LeaveSourceSpecificGroup(en0, &ssmgroup, &ssmsource); err != nil { +// // error handling +// } +// +// or JoinGroup, ExcludeSourceSpecificGroup, +// IncludeSourceSpecificGroup and LeaveGroup for the operation known +// as "exclude" mode. +// +// exclsource := net.UDPAddr{IP: net.ParseIP("fe80::dead")} +// if err := p.JoinGroup(en0, &ssmgroup); err != nil { +// // error handling +// } +// if err := p.ExcludeSourceSpecificGroup(en0, &ssmgroup, &exclsource); err != nil { +// // error handling +// } +// if err := p.LeaveGroup(en0, &ssmgroup); err != nil { +// // error handling +// } +// +// Note that it depends on each platform implementation what happens +// when an application which runs on MLDv2 unsupported platform uses +// JoinSourceSpecificGroup and LeaveSourceSpecificGroup. +// In general the platform tries to fall back to conversations using +// MLDv1 and starts to listen to multicast traffic. +// In the fallback case, ExcludeSourceSpecificGroup and +// IncludeSourceSpecificGroup may return an error. +package ipv6 // import "golang.org/x/net/ipv6" + +// BUG(mikio): This package is not implemented on NaCl and Plan 9. diff --git a/vendor/golang.org/x/net/ipv6/endpoint.go b/vendor/golang.org/x/net/ipv6/endpoint.go new file mode 100644 index 00000000..0624c174 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/endpoint.go @@ -0,0 +1,128 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "syscall" + "time" + + "golang.org/x/net/internal/socket" +) + +// BUG(mikio): On Windows, the JoinSourceSpecificGroup, +// LeaveSourceSpecificGroup, ExcludeSourceSpecificGroup and +// IncludeSourceSpecificGroup methods of PacketConn are not +// implemented. + +// A Conn represents a network endpoint that uses IPv6 transport. +// It allows to set basic IP-level socket options such as traffic +// class and hop limit. +type Conn struct { + genericOpt +} + +type genericOpt struct { + *socket.Conn +} + +func (c *genericOpt) ok() bool { return c != nil && c.Conn != nil } + +// PathMTU returns a path MTU value for the destination associated +// with the endpoint. +func (c *Conn) PathMTU() (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + so, ok := sockOpts[ssoPathMTU] + if !ok { + return 0, errOpNoSupport + } + _, mtu, err := so.getMTUInfo(c.Conn) + if err != nil { + return 0, err + } + return mtu, nil +} + +// NewConn returns a new Conn. +func NewConn(c net.Conn) *Conn { + cc, _ := socket.NewConn(c) + return &Conn{ + genericOpt: genericOpt{Conn: cc}, + } +} + +// A PacketConn represents a packet network endpoint that uses IPv6 +// transport. It is used to control several IP-level socket options +// including IPv6 header manipulation. It also provides datagram +// based network I/O methods specific to the IPv6 and higher layer +// protocols such as OSPF, GRE, and UDP. +type PacketConn struct { + genericOpt + dgramOpt + payloadHandler +} + +type dgramOpt struct { + *socket.Conn +} + +func (c *dgramOpt) ok() bool { return c != nil && c.Conn != nil } + +// SetControlMessage allows to receive the per packet basis IP-level +// socket options. +func (c *PacketConn) SetControlMessage(cf ControlFlags, on bool) error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return setControlMessage(c.dgramOpt.Conn, &c.payloadHandler.rawOpt, cf, on) +} + +// SetDeadline sets the read and write deadlines associated with the +// endpoint. +func (c *PacketConn) SetDeadline(t time.Time) error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return c.payloadHandler.SetDeadline(t) +} + +// SetReadDeadline sets the read deadline associated with the +// endpoint. +func (c *PacketConn) SetReadDeadline(t time.Time) error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return c.payloadHandler.SetReadDeadline(t) +} + +// SetWriteDeadline sets the write deadline associated with the +// endpoint. +func (c *PacketConn) SetWriteDeadline(t time.Time) error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return c.payloadHandler.SetWriteDeadline(t) +} + +// Close closes the endpoint. +func (c *PacketConn) Close() error { + if !c.payloadHandler.ok() { + return syscall.EINVAL + } + return c.payloadHandler.Close() +} + +// NewPacketConn returns a new PacketConn using c as its underlying +// transport. +func NewPacketConn(c net.PacketConn) *PacketConn { + cc, _ := socket.NewConn(c.(net.Conn)) + return &PacketConn{ + genericOpt: genericOpt{Conn: cc}, + dgramOpt: dgramOpt{Conn: cc}, + payloadHandler: payloadHandler{PacketConn: c, Conn: cc}, + } +} diff --git a/vendor/golang.org/x/net/ipv6/gen.go b/vendor/golang.org/x/net/ipv6/gen.go new file mode 100644 index 00000000..5885664f --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/gen.go @@ -0,0 +1,199 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +//go:generate go run gen.go + +// This program generates system adaptation constants and types, +// internet protocol constants and tables by reading template files +// and IANA protocol registries. +package main + +import ( + "bytes" + "encoding/xml" + "fmt" + "go/format" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "runtime" + "strconv" + "strings" +) + +func main() { + if err := genzsys(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := geniana(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func genzsys() error { + defs := "defs_" + runtime.GOOS + ".go" + f, err := os.Open(defs) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + f.Close() + cmd := exec.Command("go", "tool", "cgo", "-godefs", defs) + b, err := cmd.Output() + if err != nil { + return err + } + b, err = format.Source(b) + if err != nil { + return err + } + zsys := "zsys_" + runtime.GOOS + ".go" + switch runtime.GOOS { + case "freebsd", "linux": + zsys = "zsys_" + runtime.GOOS + "_" + runtime.GOARCH + ".go" + } + if err := ioutil.WriteFile(zsys, b, 0644); err != nil { + return err + } + return nil +} + +var registries = []struct { + url string + parse func(io.Writer, io.Reader) error +}{ + { + "https://www.iana.org/assignments/icmpv6-parameters/icmpv6-parameters.xml", + parseICMPv6Parameters, + }, +} + +func geniana() error { + var bb bytes.Buffer + fmt.Fprintf(&bb, "// go generate gen.go\n") + fmt.Fprintf(&bb, "// Code generated by the command above; DO NOT EDIT.\n\n") + fmt.Fprintf(&bb, "package ipv6\n\n") + for _, r := range registries { + resp, err := http.Get(r.url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("got HTTP status code %v for %v\n", resp.StatusCode, r.url) + } + if err := r.parse(&bb, resp.Body); err != nil { + return err + } + fmt.Fprintf(&bb, "\n") + } + b, err := format.Source(bb.Bytes()) + if err != nil { + return err + } + if err := ioutil.WriteFile("iana.go", b, 0644); err != nil { + return err + } + return nil +} + +func parseICMPv6Parameters(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var icp icmpv6Parameters + if err := dec.Decode(&icp); err != nil { + return err + } + prs := icp.escape() + fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated) + fmt.Fprintf(w, "const (\n") + for _, pr := range prs { + if pr.Name == "" { + continue + } + fmt.Fprintf(w, "ICMPType%s ICMPType = %d", pr.Name, pr.Value) + fmt.Fprintf(w, "// %s\n", pr.OrigName) + } + fmt.Fprintf(w, ")\n\n") + fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated) + fmt.Fprintf(w, "var icmpTypes = map[ICMPType]string{\n") + for _, pr := range prs { + if pr.Name == "" { + continue + } + fmt.Fprintf(w, "%d: %q,\n", pr.Value, strings.ToLower(pr.OrigName)) + } + fmt.Fprintf(w, "}\n") + return nil +} + +type icmpv6Parameters struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + Registries []struct { + Title string `xml:"title"` + Records []struct { + Value string `xml:"value"` + Name string `xml:"name"` + } `xml:"record"` + } `xml:"registry"` +} + +type canonICMPv6ParamRecord struct { + OrigName string + Name string + Value int +} + +func (icp *icmpv6Parameters) escape() []canonICMPv6ParamRecord { + id := -1 + for i, r := range icp.Registries { + if strings.Contains(r.Title, "Type") || strings.Contains(r.Title, "type") { + id = i + break + } + } + if id < 0 { + return nil + } + prs := make([]canonICMPv6ParamRecord, len(icp.Registries[id].Records)) + sr := strings.NewReplacer( + "Messages", "", + "Message", "", + "ICMP", "", + "+", "P", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, pr := range icp.Registries[id].Records { + if strings.Contains(pr.Name, "Reserved") || + strings.Contains(pr.Name, "Unassigned") || + strings.Contains(pr.Name, "Deprecated") || + strings.Contains(pr.Name, "Experiment") || + strings.Contains(pr.Name, "experiment") { + continue + } + ss := strings.Split(pr.Name, "\n") + if len(ss) > 1 { + prs[i].Name = strings.Join(ss, " ") + } else { + prs[i].Name = ss[0] + } + s := strings.TrimSpace(prs[i].Name) + prs[i].OrigName = s + prs[i].Name = sr.Replace(s) + prs[i].Value, _ = strconv.Atoi(pr.Value) + } + return prs +} diff --git a/vendor/golang.org/x/net/ipv6/genericopt.go b/vendor/golang.org/x/net/ipv6/genericopt.go new file mode 100644 index 00000000..e9dbc2e1 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/genericopt.go @@ -0,0 +1,58 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import "syscall" + +// TrafficClass returns the traffic class field value for outgoing +// packets. +func (c *genericOpt) TrafficClass() (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + so, ok := sockOpts[ssoTrafficClass] + if !ok { + return 0, errOpNoSupport + } + return so.GetInt(c.Conn) +} + +// SetTrafficClass sets the traffic class field value for future +// outgoing packets. +func (c *genericOpt) SetTrafficClass(tclass int) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoTrafficClass] + if !ok { + return errOpNoSupport + } + return so.SetInt(c.Conn, tclass) +} + +// HopLimit returns the hop limit field value for outgoing packets. +func (c *genericOpt) HopLimit() (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + so, ok := sockOpts[ssoHopLimit] + if !ok { + return 0, errOpNoSupport + } + return so.GetInt(c.Conn) +} + +// SetHopLimit sets the hop limit field value for future outgoing +// packets. +func (c *genericOpt) SetHopLimit(hoplim int) error { + if !c.ok() { + return syscall.EINVAL + } + so, ok := sockOpts[ssoHopLimit] + if !ok { + return errOpNoSupport + } + return so.SetInt(c.Conn, hoplim) +} diff --git a/vendor/golang.org/x/net/ipv6/header.go b/vendor/golang.org/x/net/ipv6/header.go new file mode 100644 index 00000000..e05cb08b --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/header.go @@ -0,0 +1,55 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "encoding/binary" + "fmt" + "net" +) + +const ( + Version = 6 // protocol version + HeaderLen = 40 // header length +) + +// A Header represents an IPv6 base header. +type Header struct { + Version int // protocol version + TrafficClass int // traffic class + FlowLabel int // flow label + PayloadLen int // payload length + NextHeader int // next header + HopLimit int // hop limit + Src net.IP // source address + Dst net.IP // destination address +} + +func (h *Header) String() string { + if h == nil { + return "" + } + return fmt.Sprintf("ver=%d tclass=%#x flowlbl=%#x payloadlen=%d nxthdr=%d hoplim=%d src=%v dst=%v", h.Version, h.TrafficClass, h.FlowLabel, h.PayloadLen, h.NextHeader, h.HopLimit, h.Src, h.Dst) +} + +// ParseHeader parses b as an IPv6 base header. +func ParseHeader(b []byte) (*Header, error) { + if len(b) < HeaderLen { + return nil, errHeaderTooShort + } + h := &Header{ + Version: int(b[0]) >> 4, + TrafficClass: int(b[0]&0x0f)<<4 | int(b[1])>>4, + FlowLabel: int(b[1]&0x0f)<<16 | int(b[2])<<8 | int(b[3]), + PayloadLen: int(binary.BigEndian.Uint16(b[4:6])), + NextHeader: int(b[6]), + HopLimit: int(b[7]), + } + h.Src = make(net.IP, net.IPv6len) + copy(h.Src, b[8:24]) + h.Dst = make(net.IP, net.IPv6len) + copy(h.Dst, b[24:40]) + return h, nil +} diff --git a/vendor/golang.org/x/net/ipv6/helper.go b/vendor/golang.org/x/net/ipv6/helper.go new file mode 100644 index 00000000..25974013 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/helper.go @@ -0,0 +1,57 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "errors" + "net" +) + +var ( + errMissingAddress = errors.New("missing address") + errHeaderTooShort = errors.New("header too short") + errInvalidConnType = errors.New("invalid conn type") + errOpNoSupport = errors.New("operation not supported") + errNoSuchInterface = errors.New("no such interface") +) + +func boolint(b bool) int { + if b { + return 1 + } + return 0 +} + +func netAddrToIP16(a net.Addr) net.IP { + switch v := a.(type) { + case *net.UDPAddr: + if ip := v.IP.To16(); ip != nil && ip.To4() == nil { + return ip + } + case *net.IPAddr: + if ip := v.IP.To16(); ip != nil && ip.To4() == nil { + return ip + } + } + return nil +} + +func opAddr(a net.Addr) net.Addr { + switch a.(type) { + case *net.TCPAddr: + if a == nil { + return nil + } + case *net.UDPAddr: + if a == nil { + return nil + } + case *net.IPAddr: + if a == nil { + return nil + } + } + return a +} diff --git a/vendor/golang.org/x/net/ipv6/iana.go b/vendor/golang.org/x/net/ipv6/iana.go new file mode 100644 index 00000000..32db1aa9 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/iana.go @@ -0,0 +1,86 @@ +// go generate gen.go +// Code generated by the command above; DO NOT EDIT. + +package ipv6 + +// Internet Control Message Protocol version 6 (ICMPv6) Parameters, Updated: 2018-03-09 +const ( + ICMPTypeDestinationUnreachable ICMPType = 1 // Destination Unreachable + ICMPTypePacketTooBig ICMPType = 2 // Packet Too Big + ICMPTypeTimeExceeded ICMPType = 3 // Time Exceeded + ICMPTypeParameterProblem ICMPType = 4 // Parameter Problem + ICMPTypeEchoRequest ICMPType = 128 // Echo Request + ICMPTypeEchoReply ICMPType = 129 // Echo Reply + ICMPTypeMulticastListenerQuery ICMPType = 130 // Multicast Listener Query + ICMPTypeMulticastListenerReport ICMPType = 131 // Multicast Listener Report + ICMPTypeMulticastListenerDone ICMPType = 132 // Multicast Listener Done + ICMPTypeRouterSolicitation ICMPType = 133 // Router Solicitation + ICMPTypeRouterAdvertisement ICMPType = 134 // Router Advertisement + ICMPTypeNeighborSolicitation ICMPType = 135 // Neighbor Solicitation + ICMPTypeNeighborAdvertisement ICMPType = 136 // Neighbor Advertisement + ICMPTypeRedirect ICMPType = 137 // Redirect Message + ICMPTypeRouterRenumbering ICMPType = 138 // Router Renumbering + ICMPTypeNodeInformationQuery ICMPType = 139 // ICMP Node Information Query + ICMPTypeNodeInformationResponse ICMPType = 140 // ICMP Node Information Response + ICMPTypeInverseNeighborDiscoverySolicitation ICMPType = 141 // Inverse Neighbor Discovery Solicitation Message + ICMPTypeInverseNeighborDiscoveryAdvertisement ICMPType = 142 // Inverse Neighbor Discovery Advertisement Message + ICMPTypeVersion2MulticastListenerReport ICMPType = 143 // Version 2 Multicast Listener Report + ICMPTypeHomeAgentAddressDiscoveryRequest ICMPType = 144 // Home Agent Address Discovery Request Message + ICMPTypeHomeAgentAddressDiscoveryReply ICMPType = 145 // Home Agent Address Discovery Reply Message + ICMPTypeMobilePrefixSolicitation ICMPType = 146 // Mobile Prefix Solicitation + ICMPTypeMobilePrefixAdvertisement ICMPType = 147 // Mobile Prefix Advertisement + ICMPTypeCertificationPathSolicitation ICMPType = 148 // Certification Path Solicitation Message + ICMPTypeCertificationPathAdvertisement ICMPType = 149 // Certification Path Advertisement Message + ICMPTypeMulticastRouterAdvertisement ICMPType = 151 // Multicast Router Advertisement + ICMPTypeMulticastRouterSolicitation ICMPType = 152 // Multicast Router Solicitation + ICMPTypeMulticastRouterTermination ICMPType = 153 // Multicast Router Termination + ICMPTypeFMIPv6 ICMPType = 154 // FMIPv6 Messages + ICMPTypeRPLControl ICMPType = 155 // RPL Control Message + ICMPTypeILNPv6LocatorUpdate ICMPType = 156 // ILNPv6 Locator Update Message + ICMPTypeDuplicateAddressRequest ICMPType = 157 // Duplicate Address Request + ICMPTypeDuplicateAddressConfirmation ICMPType = 158 // Duplicate Address Confirmation + ICMPTypeMPLControl ICMPType = 159 // MPL Control Message + ICMPTypeExtendedEchoRequest ICMPType = 160 // Extended Echo Request + ICMPTypeExtendedEchoReply ICMPType = 161 // Extended Echo Reply +) + +// Internet Control Message Protocol version 6 (ICMPv6) Parameters, Updated: 2018-03-09 +var icmpTypes = map[ICMPType]string{ + 1: "destination unreachable", + 2: "packet too big", + 3: "time exceeded", + 4: "parameter problem", + 128: "echo request", + 129: "echo reply", + 130: "multicast listener query", + 131: "multicast listener report", + 132: "multicast listener done", + 133: "router solicitation", + 134: "router advertisement", + 135: "neighbor solicitation", + 136: "neighbor advertisement", + 137: "redirect message", + 138: "router renumbering", + 139: "icmp node information query", + 140: "icmp node information response", + 141: "inverse neighbor discovery solicitation message", + 142: "inverse neighbor discovery advertisement message", + 143: "version 2 multicast listener report", + 144: "home agent address discovery request message", + 145: "home agent address discovery reply message", + 146: "mobile prefix solicitation", + 147: "mobile prefix advertisement", + 148: "certification path solicitation message", + 149: "certification path advertisement message", + 151: "multicast router advertisement", + 152: "multicast router solicitation", + 153: "multicast router termination", + 154: "fmipv6 messages", + 155: "rpl control message", + 156: "ilnpv6 locator update message", + 157: "duplicate address request", + 158: "duplicate address confirmation", + 159: "mpl control message", + 160: "extended echo request", + 161: "extended echo reply", +} diff --git a/vendor/golang.org/x/net/ipv6/icmp.go b/vendor/golang.org/x/net/ipv6/icmp.go new file mode 100644 index 00000000..b7f48e27 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/icmp.go @@ -0,0 +1,60 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import "golang.org/x/net/internal/iana" + +// BUG(mikio): On Windows, methods related to ICMPFilter are not +// implemented. + +// An ICMPType represents a type of ICMP message. +type ICMPType int + +func (typ ICMPType) String() string { + s, ok := icmpTypes[typ] + if !ok { + return "" + } + return s +} + +// Protocol returns the ICMPv6 protocol number. +func (typ ICMPType) Protocol() int { + return iana.ProtocolIPv6ICMP +} + +// An ICMPFilter represents an ICMP message filter for incoming +// packets. The filter belongs to a packet delivery path on a host and +// it cannot interact with forwarding packets or tunnel-outer packets. +// +// Note: RFC 8200 defines a reasonable role model. A node means a +// device that implements IP. A router means a node that forwards IP +// packets not explicitly addressed to itself, and a host means a node +// that is not a router. +type ICMPFilter struct { + icmpv6Filter +} + +// Accept accepts incoming ICMP packets including the type field value +// typ. +func (f *ICMPFilter) Accept(typ ICMPType) { + f.accept(typ) +} + +// Block blocks incoming ICMP packets including the type field value +// typ. +func (f *ICMPFilter) Block(typ ICMPType) { + f.block(typ) +} + +// SetAll sets the filter action to the filter. +func (f *ICMPFilter) SetAll(block bool) { + f.setAll(block) +} + +// WillBlock reports whether the ICMP type will be blocked. +func (f *ICMPFilter) WillBlock(typ ICMPType) bool { + return f.willBlock(typ) +} diff --git a/vendor/golang.org/x/net/ipv6/icmp_bsd.go b/vendor/golang.org/x/net/ipv6/icmp_bsd.go new file mode 100644 index 00000000..e1a791de --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/icmp_bsd.go @@ -0,0 +1,29 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd + +package ipv6 + +func (f *icmpv6Filter) accept(typ ICMPType) { + f.Filt[typ>>5] |= 1 << (uint32(typ) & 31) +} + +func (f *icmpv6Filter) block(typ ICMPType) { + f.Filt[typ>>5] &^= 1 << (uint32(typ) & 31) +} + +func (f *icmpv6Filter) setAll(block bool) { + for i := range f.Filt { + if block { + f.Filt[i] = 0 + } else { + f.Filt[i] = 1<<32 - 1 + } + } +} + +func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + return f.Filt[typ>>5]&(1<<(uint32(typ)&31)) == 0 +} diff --git a/vendor/golang.org/x/net/ipv6/icmp_linux.go b/vendor/golang.org/x/net/ipv6/icmp_linux.go new file mode 100644 index 00000000..647f6b44 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/icmp_linux.go @@ -0,0 +1,27 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +func (f *icmpv6Filter) accept(typ ICMPType) { + f.Data[typ>>5] &^= 1 << (uint32(typ) & 31) +} + +func (f *icmpv6Filter) block(typ ICMPType) { + f.Data[typ>>5] |= 1 << (uint32(typ) & 31) +} + +func (f *icmpv6Filter) setAll(block bool) { + for i := range f.Data { + if block { + f.Data[i] = 1<<32 - 1 + } else { + f.Data[i] = 0 + } + } +} + +func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + return f.Data[typ>>5]&(1<<(uint32(typ)&31)) != 0 +} diff --git a/vendor/golang.org/x/net/ipv6/icmp_solaris.go b/vendor/golang.org/x/net/ipv6/icmp_solaris.go new file mode 100644 index 00000000..7c23bb1c --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/icmp_solaris.go @@ -0,0 +1,27 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +func (f *icmpv6Filter) accept(typ ICMPType) { + f.X__icmp6_filt[typ>>5] |= 1 << (uint32(typ) & 31) +} + +func (f *icmpv6Filter) block(typ ICMPType) { + f.X__icmp6_filt[typ>>5] &^= 1 << (uint32(typ) & 31) +} + +func (f *icmpv6Filter) setAll(block bool) { + for i := range f.X__icmp6_filt { + if block { + f.X__icmp6_filt[i] = 0 + } else { + f.X__icmp6_filt[i] = 1<<32 - 1 + } + } +} + +func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + return f.X__icmp6_filt[typ>>5]&(1<<(uint32(typ)&31)) == 0 +} diff --git a/vendor/golang.org/x/net/ipv6/icmp_stub.go b/vendor/golang.org/x/net/ipv6/icmp_stub.go new file mode 100644 index 00000000..c4b9be6d --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/icmp_stub.go @@ -0,0 +1,23 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package ipv6 + +type icmpv6Filter struct { +} + +func (f *icmpv6Filter) accept(typ ICMPType) { +} + +func (f *icmpv6Filter) block(typ ICMPType) { +} + +func (f *icmpv6Filter) setAll(block bool) { +} + +func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + return false +} diff --git a/vendor/golang.org/x/net/ipv6/icmp_windows.go b/vendor/golang.org/x/net/ipv6/icmp_windows.go new file mode 100644 index 00000000..443cd073 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/icmp_windows.go @@ -0,0 +1,22 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +func (f *icmpv6Filter) accept(typ ICMPType) { + // TODO(mikio): implement this +} + +func (f *icmpv6Filter) block(typ ICMPType) { + // TODO(mikio): implement this +} + +func (f *icmpv6Filter) setAll(block bool) { + // TODO(mikio): implement this +} + +func (f *icmpv6Filter) willBlock(typ ICMPType) bool { + // TODO(mikio): implement this + return false +} diff --git a/vendor/golang.org/x/net/ipv6/payload.go b/vendor/golang.org/x/net/ipv6/payload.go new file mode 100644 index 00000000..a8197f16 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/payload.go @@ -0,0 +1,23 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +// BUG(mikio): On Windows, the ControlMessage for ReadFrom and WriteTo +// methods of PacketConn is not implemented. + +// A payloadHandler represents the IPv6 datagram payload handler. +type payloadHandler struct { + net.PacketConn + *socket.Conn + rawOpt +} + +func (c *payloadHandler) ok() bool { return c != nil && c.PacketConn != nil && c.Conn != nil } diff --git a/vendor/golang.org/x/net/ipv6/payload_cmsg.go b/vendor/golang.org/x/net/ipv6/payload_cmsg.go new file mode 100644 index 00000000..4ee4b062 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/payload_cmsg.go @@ -0,0 +1,35 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !nacl,!plan9,!windows + +package ipv6 + +import ( + "net" + "syscall" +) + +// ReadFrom reads a payload of the received IPv6 datagram, from the +// endpoint c, copying the payload into b. It returns the number of +// bytes copied into b, the control message cm and the source address +// src of the received datagram. +func (c *payloadHandler) ReadFrom(b []byte) (n int, cm *ControlMessage, src net.Addr, err error) { + if !c.ok() { + return 0, nil, nil, syscall.EINVAL + } + return c.readFrom(b) +} + +// WriteTo writes a payload of the IPv6 datagram, to the destination +// address dst through the endpoint c, copying the payload from b. It +// returns the number of bytes written. The control message cm allows +// the IPv6 header fields and the datagram path to be specified. The +// cm may be nil if control of the outgoing datagram is not required. +func (c *payloadHandler) WriteTo(b []byte, cm *ControlMessage, dst net.Addr) (n int, err error) { + if !c.ok() { + return 0, syscall.EINVAL + } + return c.writeTo(b, cm, dst) +} diff --git a/vendor/golang.org/x/net/ipv6/payload_cmsg_go1_8.go b/vendor/golang.org/x/net/ipv6/payload_cmsg_go1_8.go new file mode 100644 index 00000000..fdc6c399 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/payload_cmsg_go1_8.go @@ -0,0 +1,55 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.9 +// +build !nacl,!plan9,!windows + +package ipv6 + +import "net" + +func (c *payloadHandler) readFrom(b []byte) (n int, cm *ControlMessage, src net.Addr, err error) { + c.rawOpt.RLock() + oob := NewControlMessage(c.rawOpt.cflags) + c.rawOpt.RUnlock() + var nn int + switch c := c.PacketConn.(type) { + case *net.UDPConn: + if n, nn, _, src, err = c.ReadMsgUDP(b, oob); err != nil { + return 0, nil, nil, err + } + case *net.IPConn: + if n, nn, _, src, err = c.ReadMsgIP(b, oob); err != nil { + return 0, nil, nil, err + } + default: + return 0, nil, nil, &net.OpError{Op: "read", Net: c.LocalAddr().Network(), Source: c.LocalAddr(), Err: errInvalidConnType} + } + if nn > 0 { + cm = new(ControlMessage) + if err = cm.Parse(oob[:nn]); err != nil { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + } + if cm != nil { + cm.Src = netAddrToIP16(src) + } + return +} + +func (c *payloadHandler) writeTo(b []byte, cm *ControlMessage, dst net.Addr) (n int, err error) { + oob := cm.Marshal() + if dst == nil { + return 0, &net.OpError{Op: "write", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: errMissingAddress} + } + switch c := c.PacketConn.(type) { + case *net.UDPConn: + n, _, err = c.WriteMsgUDP(b, oob, dst.(*net.UDPAddr)) + case *net.IPConn: + n, _, err = c.WriteMsgIP(b, oob, dst.(*net.IPAddr)) + default: + return 0, &net.OpError{Op: "write", Net: c.LocalAddr().Network(), Source: c.LocalAddr(), Addr: opAddr(dst), Err: errInvalidConnType} + } + return +} diff --git a/vendor/golang.org/x/net/ipv6/payload_cmsg_go1_9.go b/vendor/golang.org/x/net/ipv6/payload_cmsg_go1_9.go new file mode 100644 index 00000000..8f6d02e2 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/payload_cmsg_go1_9.go @@ -0,0 +1,57 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.9 +// +build !nacl,!plan9,!windows + +package ipv6 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +func (c *payloadHandler) readFrom(b []byte) (int, *ControlMessage, net.Addr, error) { + c.rawOpt.RLock() + m := socket.Message{ + Buffers: [][]byte{b}, + OOB: NewControlMessage(c.rawOpt.cflags), + } + c.rawOpt.RUnlock() + switch c.PacketConn.(type) { + case *net.UDPConn: + if err := c.RecvMsg(&m, 0); err != nil { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + case *net.IPConn: + if err := c.RecvMsg(&m, 0); err != nil { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + default: + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: errInvalidConnType} + } + var cm *ControlMessage + if m.NN > 0 { + cm = new(ControlMessage) + if err := cm.Parse(m.OOB[:m.NN]); err != nil { + return 0, nil, nil, &net.OpError{Op: "read", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Err: err} + } + cm.Src = netAddrToIP16(m.Addr) + } + return m.N, cm, m.Addr, nil +} + +func (c *payloadHandler) writeTo(b []byte, cm *ControlMessage, dst net.Addr) (int, error) { + m := socket.Message{ + Buffers: [][]byte{b}, + OOB: cm.Marshal(), + Addr: dst, + } + err := c.SendMsg(&m, 0) + if err != nil { + err = &net.OpError{Op: "write", Net: c.PacketConn.LocalAddr().Network(), Source: c.PacketConn.LocalAddr(), Addr: opAddr(dst), Err: err} + } + return m.N, err +} diff --git a/vendor/golang.org/x/net/ipv6/payload_nocmsg.go b/vendor/golang.org/x/net/ipv6/payload_nocmsg.go new file mode 100644 index 00000000..99a43542 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/payload_nocmsg.go @@ -0,0 +1,41 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build nacl plan9 windows + +package ipv6 + +import ( + "net" + "syscall" +) + +// ReadFrom reads a payload of the received IPv6 datagram, from the +// endpoint c, copying the payload into b. It returns the number of +// bytes copied into b, the control message cm and the source address +// src of the received datagram. +func (c *payloadHandler) ReadFrom(b []byte) (n int, cm *ControlMessage, src net.Addr, err error) { + if !c.ok() { + return 0, nil, nil, syscall.EINVAL + } + if n, src, err = c.PacketConn.ReadFrom(b); err != nil { + return 0, nil, nil, err + } + return +} + +// WriteTo writes a payload of the IPv6 datagram, to the destination +// address dst through the endpoint c, copying the payload from b. It +// returns the number of bytes written. The control message cm allows +// the IPv6 header fields and the datagram path to be specified. The +// cm may be nil if control of the outgoing datagram is not required. +func (c *payloadHandler) WriteTo(b []byte, cm *ControlMessage, dst net.Addr) (n int, err error) { + if !c.ok() { + return 0, syscall.EINVAL + } + if dst == nil { + return 0, errMissingAddress + } + return c.PacketConn.WriteTo(b, dst) +} diff --git a/vendor/golang.org/x/net/ipv6/sockopt.go b/vendor/golang.org/x/net/ipv6/sockopt.go new file mode 100644 index 00000000..cc3907df --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sockopt.go @@ -0,0 +1,43 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import "golang.org/x/net/internal/socket" + +// Sticky socket options +const ( + ssoTrafficClass = iota // header field for unicast packet, RFC 3542 + ssoHopLimit // header field for unicast packet, RFC 3493 + ssoMulticastInterface // outbound interface for multicast packet, RFC 3493 + ssoMulticastHopLimit // header field for multicast packet, RFC 3493 + ssoMulticastLoopback // loopback for multicast packet, RFC 3493 + ssoReceiveTrafficClass // header field on received packet, RFC 3542 + ssoReceiveHopLimit // header field on received packet, RFC 2292 or 3542 + ssoReceivePacketInfo // incbound or outbound packet path, RFC 2292 or 3542 + ssoReceivePathMTU // path mtu, RFC 3542 + ssoPathMTU // path mtu, RFC 3542 + ssoChecksum // packet checksum, RFC 2292 or 3542 + ssoICMPFilter // icmp filter, RFC 2292 or 3542 + ssoJoinGroup // any-source multicast, RFC 3493 + ssoLeaveGroup // any-source multicast, RFC 3493 + ssoJoinSourceGroup // source-specific multicast + ssoLeaveSourceGroup // source-specific multicast + ssoBlockSourceGroup // any-source or source-specific multicast + ssoUnblockSourceGroup // any-source or source-specific multicast + ssoAttachFilter // attach BPF for filtering inbound traffic +) + +// Sticky socket option value types +const ( + ssoTypeIPMreq = iota + 1 + ssoTypeGroupReq + ssoTypeGroupSourceReq +) + +// A sockOpt represents a binding for sticky socket option. +type sockOpt struct { + socket.Option + typ int // hint for option value type; optional +} diff --git a/vendor/golang.org/x/net/ipv6/sockopt_posix.go b/vendor/golang.org/x/net/ipv6/sockopt_posix.go new file mode 100644 index 00000000..0eac86eb --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sockopt_posix.go @@ -0,0 +1,87 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris windows + +package ipv6 + +import ( + "net" + "unsafe" + + "golang.org/x/net/bpf" + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) getMulticastInterface(c *socket.Conn) (*net.Interface, error) { + n, err := so.GetInt(c) + if err != nil { + return nil, err + } + return net.InterfaceByIndex(n) +} + +func (so *sockOpt) setMulticastInterface(c *socket.Conn, ifi *net.Interface) error { + var n int + if ifi != nil { + n = ifi.Index + } + return so.SetInt(c, n) +} + +func (so *sockOpt) getICMPFilter(c *socket.Conn) (*ICMPFilter, error) { + b := make([]byte, so.Len) + n, err := so.Get(c, b) + if err != nil { + return nil, err + } + if n != sizeofICMPv6Filter { + return nil, errOpNoSupport + } + return (*ICMPFilter)(unsafe.Pointer(&b[0])), nil +} + +func (so *sockOpt) setICMPFilter(c *socket.Conn, f *ICMPFilter) error { + b := (*[sizeofICMPv6Filter]byte)(unsafe.Pointer(f))[:sizeofICMPv6Filter] + return so.Set(c, b) +} + +func (so *sockOpt) getMTUInfo(c *socket.Conn) (*net.Interface, int, error) { + b := make([]byte, so.Len) + n, err := so.Get(c, b) + if err != nil { + return nil, 0, err + } + if n != sizeofIPv6Mtuinfo { + return nil, 0, errOpNoSupport + } + mi := (*ipv6Mtuinfo)(unsafe.Pointer(&b[0])) + if mi.Addr.Scope_id == 0 { + return nil, int(mi.Mtu), nil + } + ifi, err := net.InterfaceByIndex(int(mi.Addr.Scope_id)) + if err != nil { + return nil, 0, err + } + return ifi, int(mi.Mtu), nil +} + +func (so *sockOpt) setGroup(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + switch so.typ { + case ssoTypeIPMreq: + return so.setIPMreq(c, ifi, grp) + case ssoTypeGroupReq: + return so.setGroupReq(c, ifi, grp) + default: + return errOpNoSupport + } +} + +func (so *sockOpt) setSourceGroup(c *socket.Conn, ifi *net.Interface, grp, src net.IP) error { + return so.setGroupSourceReq(c, ifi, grp, src) +} + +func (so *sockOpt) setBPF(c *socket.Conn, f []bpf.RawInstruction) error { + return so.setAttachFilter(c, f) +} diff --git a/vendor/golang.org/x/net/ipv6/sockopt_stub.go b/vendor/golang.org/x/net/ipv6/sockopt_stub.go new file mode 100644 index 00000000..1f4a273e --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sockopt_stub.go @@ -0,0 +1,46 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package ipv6 + +import ( + "net" + + "golang.org/x/net/bpf" + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) getMulticastInterface(c *socket.Conn) (*net.Interface, error) { + return nil, errOpNoSupport +} + +func (so *sockOpt) setMulticastInterface(c *socket.Conn, ifi *net.Interface) error { + return errOpNoSupport +} + +func (so *sockOpt) getICMPFilter(c *socket.Conn) (*ICMPFilter, error) { + return nil, errOpNoSupport +} + +func (so *sockOpt) setICMPFilter(c *socket.Conn, f *ICMPFilter) error { + return errOpNoSupport +} + +func (so *sockOpt) getMTUInfo(c *socket.Conn) (*net.Interface, int, error) { + return nil, 0, errOpNoSupport +} + +func (so *sockOpt) setGroup(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + return errOpNoSupport +} + +func (so *sockOpt) setSourceGroup(c *socket.Conn, ifi *net.Interface, grp, src net.IP) error { + return errOpNoSupport +} + +func (so *sockOpt) setBPF(c *socket.Conn, f []bpf.RawInstruction) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv6/sys_asmreq.go b/vendor/golang.org/x/net/ipv6/sys_asmreq.go new file mode 100644 index 00000000..b0510c0b --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_asmreq.go @@ -0,0 +1,24 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris windows + +package ipv6 + +import ( + "net" + "unsafe" + + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setIPMreq(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + var mreq ipv6Mreq + copy(mreq.Multiaddr[:], grp) + if ifi != nil { + mreq.setIfindex(ifi.Index) + } + b := (*[sizeofIPv6Mreq]byte)(unsafe.Pointer(&mreq))[:sizeofIPv6Mreq] + return so.Set(c, b) +} diff --git a/vendor/golang.org/x/net/ipv6/sys_asmreq_stub.go b/vendor/golang.org/x/net/ipv6/sys_asmreq_stub.go new file mode 100644 index 00000000..eece9618 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_asmreq_stub.go @@ -0,0 +1,17 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package ipv6 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setIPMreq(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv6/sys_bpf.go b/vendor/golang.org/x/net/ipv6/sys_bpf.go new file mode 100644 index 00000000..b2dbcb2f --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_bpf.go @@ -0,0 +1,23 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package ipv6 + +import ( + "unsafe" + + "golang.org/x/net/bpf" + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setAttachFilter(c *socket.Conn, f []bpf.RawInstruction) error { + prog := sockFProg{ + Len: uint16(len(f)), + Filter: (*sockFilter)(unsafe.Pointer(&f[0])), + } + b := (*[sizeofSockFprog]byte)(unsafe.Pointer(&prog))[:sizeofSockFprog] + return so.Set(c, b) +} diff --git a/vendor/golang.org/x/net/ipv6/sys_bpf_stub.go b/vendor/golang.org/x/net/ipv6/sys_bpf_stub.go new file mode 100644 index 00000000..676bea55 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_bpf_stub.go @@ -0,0 +1,16 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !linux + +package ipv6 + +import ( + "golang.org/x/net/bpf" + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setAttachFilter(c *socket.Conn, f []bpf.RawInstruction) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv6/sys_bsd.go b/vendor/golang.org/x/net/ipv6/sys_bsd.go new file mode 100644 index 00000000..e416eaa1 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_bsd.go @@ -0,0 +1,57 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build dragonfly netbsd openbsd + +package ipv6 + +import ( + "net" + "syscall" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTrafficClass: {sysIPV6_TCLASS, 4, marshalTrafficClass, parseTrafficClass}, + ctlHopLimit: {sysIPV6_HOPLIMIT, 4, marshalHopLimit, parseHopLimit}, + ctlPacketInfo: {sysIPV6_PKTINFO, sizeofInet6Pktinfo, marshalPacketInfo, parsePacketInfo}, + ctlNextHop: {sysIPV6_NEXTHOP, sizeofSockaddrInet6, marshalNextHop, parseNextHop}, + ctlPathMTU: {sysIPV6_PATHMTU, sizeofIPv6Mtuinfo, marshalPathMTU, parsePathMTU}, + } + + sockOpts = map[int]*sockOpt{ + ssoTrafficClass: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_TCLASS, Len: 4}}, + ssoHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_UNICAST_HOPS, Len: 4}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_IF, Len: 4}}, + ssoMulticastHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_HOPS, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_LOOP, Len: 4}}, + ssoReceiveTrafficClass: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVTCLASS, Len: 4}}, + ssoReceiveHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVHOPLIMIT, Len: 4}}, + ssoReceivePacketInfo: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPKTINFO, Len: 4}}, + ssoReceivePathMTU: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPATHMTU, Len: 4}}, + ssoPathMTU: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_PATHMTU, Len: sizeofIPv6Mtuinfo}}, + ssoChecksum: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_CHECKSUM, Len: 4}}, + ssoICMPFilter: {Option: socket.Option{Level: iana.ProtocolIPv6ICMP, Name: sysICMP6_FILTER, Len: sizeofICMPv6Filter}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_JOIN_GROUP, Len: sizeofIPv6Mreq}, typ: ssoTypeIPMreq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_LEAVE_GROUP, Len: sizeofIPv6Mreq}, typ: ssoTypeIPMreq}, + } +) + +func (sa *sockaddrInet6) setSockaddr(ip net.IP, i int) { + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(i) +} + +func (pi *inet6Pktinfo) setIfindex(i int) { + pi.Ifindex = uint32(i) +} + +func (mreq *ipv6Mreq) setIfindex(i int) { + mreq.Interface = uint32(i) +} diff --git a/vendor/golang.org/x/net/ipv6/sys_darwin.go b/vendor/golang.org/x/net/ipv6/sys_darwin.go new file mode 100644 index 00000000..e3d04439 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_darwin.go @@ -0,0 +1,106 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "strconv" + "strings" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlHopLimit: {sysIPV6_2292HOPLIMIT, 4, marshal2292HopLimit, parseHopLimit}, + ctlPacketInfo: {sysIPV6_2292PKTINFO, sizeofInet6Pktinfo, marshal2292PacketInfo, parsePacketInfo}, + } + + sockOpts = map[int]*sockOpt{ + ssoHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_UNICAST_HOPS, Len: 4}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_IF, Len: 4}}, + ssoMulticastHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_HOPS, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_LOOP, Len: 4}}, + ssoReceiveHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_2292HOPLIMIT, Len: 4}}, + ssoReceivePacketInfo: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_2292PKTINFO, Len: 4}}, + ssoChecksum: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_CHECKSUM, Len: 4}}, + ssoICMPFilter: {Option: socket.Option{Level: iana.ProtocolIPv6ICMP, Name: sysICMP6_FILTER, Len: sizeofICMPv6Filter}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_JOIN_GROUP, Len: sizeofIPv6Mreq}, typ: ssoTypeIPMreq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_LEAVE_GROUP, Len: sizeofIPv6Mreq}, typ: ssoTypeIPMreq}, + } +) + +func init() { + // Seems like kern.osreldate is veiled on latest OS X. We use + // kern.osrelease instead. + s, err := syscall.Sysctl("kern.osrelease") + if err != nil { + return + } + ss := strings.Split(s, ".") + if len(ss) == 0 { + return + } + // The IP_PKTINFO and protocol-independent multicast API were + // introduced in OS X 10.7 (Darwin 11). But it looks like + // those features require OS X 10.8 (Darwin 12) or above. + // See http://support.apple.com/kb/HT1633. + if mjver, err := strconv.Atoi(ss[0]); err != nil || mjver < 12 { + return + } + ctlOpts[ctlTrafficClass] = ctlOpt{sysIPV6_TCLASS, 4, marshalTrafficClass, parseTrafficClass} + ctlOpts[ctlHopLimit] = ctlOpt{sysIPV6_HOPLIMIT, 4, marshalHopLimit, parseHopLimit} + ctlOpts[ctlPacketInfo] = ctlOpt{sysIPV6_PKTINFO, sizeofInet6Pktinfo, marshalPacketInfo, parsePacketInfo} + ctlOpts[ctlNextHop] = ctlOpt{sysIPV6_NEXTHOP, sizeofSockaddrInet6, marshalNextHop, parseNextHop} + ctlOpts[ctlPathMTU] = ctlOpt{sysIPV6_PATHMTU, sizeofIPv6Mtuinfo, marshalPathMTU, parsePathMTU} + sockOpts[ssoTrafficClass] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_TCLASS, Len: 4}} + sockOpts[ssoReceiveTrafficClass] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVTCLASS, Len: 4}} + sockOpts[ssoReceiveHopLimit] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVHOPLIMIT, Len: 4}} + sockOpts[ssoReceivePacketInfo] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPKTINFO, Len: 4}} + sockOpts[ssoReceivePathMTU] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPATHMTU, Len: 4}} + sockOpts[ssoPathMTU] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_PATHMTU, Len: sizeofIPv6Mtuinfo}} + sockOpts[ssoJoinGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_JOIN_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq} + sockOpts[ssoLeaveGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_LEAVE_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq} + sockOpts[ssoJoinSourceGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_JOIN_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq} + sockOpts[ssoLeaveSourceGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_LEAVE_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq} + sockOpts[ssoBlockSourceGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_BLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq} + sockOpts[ssoUnblockSourceGroup] = &sockOpt{Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_UNBLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq} +} + +func (sa *sockaddrInet6) setSockaddr(ip net.IP, i int) { + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(i) +} + +func (pi *inet6Pktinfo) setIfindex(i int) { + pi.Ifindex = uint32(i) +} + +func (mreq *ipv6Mreq) setIfindex(i int) { + mreq.Interface = uint32(i) +} + +func (gr *groupReq) setGroup(grp net.IP) { + sa := (*sockaddrInet6)(unsafe.Pointer(uintptr(unsafe.Pointer(gr)) + 4)) + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], grp) +} + +func (gsr *groupSourceReq) setSourceGroup(grp, src net.IP) { + sa := (*sockaddrInet6)(unsafe.Pointer(uintptr(unsafe.Pointer(gsr)) + 4)) + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], grp) + sa = (*sockaddrInet6)(unsafe.Pointer(uintptr(unsafe.Pointer(gsr)) + 132)) + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], src) +} diff --git a/vendor/golang.org/x/net/ipv6/sys_freebsd.go b/vendor/golang.org/x/net/ipv6/sys_freebsd.go new file mode 100644 index 00000000..e9349dc2 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_freebsd.go @@ -0,0 +1,92 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "runtime" + "strings" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTrafficClass: {sysIPV6_TCLASS, 4, marshalTrafficClass, parseTrafficClass}, + ctlHopLimit: {sysIPV6_HOPLIMIT, 4, marshalHopLimit, parseHopLimit}, + ctlPacketInfo: {sysIPV6_PKTINFO, sizeofInet6Pktinfo, marshalPacketInfo, parsePacketInfo}, + ctlNextHop: {sysIPV6_NEXTHOP, sizeofSockaddrInet6, marshalNextHop, parseNextHop}, + ctlPathMTU: {sysIPV6_PATHMTU, sizeofIPv6Mtuinfo, marshalPathMTU, parsePathMTU}, + } + + sockOpts = map[int]sockOpt{ + ssoTrafficClass: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_TCLASS, Len: 4}}, + ssoHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_UNICAST_HOPS, Len: 4}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_IF, Len: 4}}, + ssoMulticastHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_HOPS, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_LOOP, Len: 4}}, + ssoReceiveTrafficClass: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVTCLASS, Len: 4}}, + ssoReceiveHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVHOPLIMIT, Len: 4}}, + ssoReceivePacketInfo: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPKTINFO, Len: 4}}, + ssoReceivePathMTU: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPATHMTU, Len: 4}}, + ssoPathMTU: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_PATHMTU, Len: sizeofIPv6Mtuinfo}}, + ssoChecksum: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_CHECKSUM, Len: 4}}, + ssoICMPFilter: {Option: socket.Option{Level: iana.ProtocolIPv6ICMP, Name: sysICMP6_FILTER, Len: sizeofICMPv6Filter}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_JOIN_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_LEAVE_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoJoinSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_JOIN_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoLeaveSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_LEAVE_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoBlockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_BLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoUnblockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_UNBLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + } +) + +func init() { + if runtime.GOOS == "freebsd" && runtime.GOARCH == "386" { + archs, _ := syscall.Sysctl("kern.supported_archs") + for _, s := range strings.Fields(archs) { + if s == "amd64" { + freebsd32o64 = true + break + } + } + } +} + +func (sa *sockaddrInet6) setSockaddr(ip net.IP, i int) { + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(i) +} + +func (pi *inet6Pktinfo) setIfindex(i int) { + pi.Ifindex = uint32(i) +} + +func (mreq *ipv6Mreq) setIfindex(i int) { + mreq.Interface = uint32(i) +} + +func (gr *groupReq) setGroup(grp net.IP) { + sa := (*sockaddrInet6)(unsafe.Pointer(&gr.Group)) + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], grp) +} + +func (gsr *groupSourceReq) setSourceGroup(grp, src net.IP) { + sa := (*sockaddrInet6)(unsafe.Pointer(&gsr.Group)) + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], grp) + sa = (*sockaddrInet6)(unsafe.Pointer(&gsr.Source)) + sa.Len = sizeofSockaddrInet6 + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], src) +} diff --git a/vendor/golang.org/x/net/ipv6/sys_linux.go b/vendor/golang.org/x/net/ipv6/sys_linux.go new file mode 100644 index 00000000..bc218103 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_linux.go @@ -0,0 +1,74 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTrafficClass: {sysIPV6_TCLASS, 4, marshalTrafficClass, parseTrafficClass}, + ctlHopLimit: {sysIPV6_HOPLIMIT, 4, marshalHopLimit, parseHopLimit}, + ctlPacketInfo: {sysIPV6_PKTINFO, sizeofInet6Pktinfo, marshalPacketInfo, parsePacketInfo}, + ctlPathMTU: {sysIPV6_PATHMTU, sizeofIPv6Mtuinfo, marshalPathMTU, parsePathMTU}, + } + + sockOpts = map[int]*sockOpt{ + ssoTrafficClass: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_TCLASS, Len: 4}}, + ssoHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_UNICAST_HOPS, Len: 4}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_IF, Len: 4}}, + ssoMulticastHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_HOPS, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_LOOP, Len: 4}}, + ssoReceiveTrafficClass: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVTCLASS, Len: 4}}, + ssoReceiveHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVHOPLIMIT, Len: 4}}, + ssoReceivePacketInfo: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPKTINFO, Len: 4}}, + ssoReceivePathMTU: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPATHMTU, Len: 4}}, + ssoPathMTU: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_PATHMTU, Len: sizeofIPv6Mtuinfo}}, + ssoChecksum: {Option: socket.Option{Level: iana.ProtocolReserved, Name: sysIPV6_CHECKSUM, Len: 4}}, + ssoICMPFilter: {Option: socket.Option{Level: iana.ProtocolIPv6ICMP, Name: sysICMPV6_FILTER, Len: sizeofICMPv6Filter}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_JOIN_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_LEAVE_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoJoinSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_JOIN_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoLeaveSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_LEAVE_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoBlockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_BLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoUnblockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_UNBLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoAttachFilter: {Option: socket.Option{Level: sysSOL_SOCKET, Name: sysSO_ATTACH_FILTER, Len: sizeofSockFprog}}, + } +) + +func (sa *sockaddrInet6) setSockaddr(ip net.IP, i int) { + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(i) +} + +func (pi *inet6Pktinfo) setIfindex(i int) { + pi.Ifindex = int32(i) +} + +func (mreq *ipv6Mreq) setIfindex(i int) { + mreq.Ifindex = int32(i) +} + +func (gr *groupReq) setGroup(grp net.IP) { + sa := (*sockaddrInet6)(unsafe.Pointer(&gr.Group)) + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], grp) +} + +func (gsr *groupSourceReq) setSourceGroup(grp, src net.IP) { + sa := (*sockaddrInet6)(unsafe.Pointer(&gsr.Group)) + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], grp) + sa = (*sockaddrInet6)(unsafe.Pointer(&gsr.Source)) + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], src) +} diff --git a/vendor/golang.org/x/net/ipv6/sys_solaris.go b/vendor/golang.org/x/net/ipv6/sys_solaris.go new file mode 100644 index 00000000..d348b5f6 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_solaris.go @@ -0,0 +1,74 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "syscall" + "unsafe" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +var ( + ctlOpts = [ctlMax]ctlOpt{ + ctlTrafficClass: {sysIPV6_TCLASS, 4, marshalTrafficClass, parseTrafficClass}, + ctlHopLimit: {sysIPV6_HOPLIMIT, 4, marshalHopLimit, parseHopLimit}, + ctlPacketInfo: {sysIPV6_PKTINFO, sizeofInet6Pktinfo, marshalPacketInfo, parsePacketInfo}, + ctlNextHop: {sysIPV6_NEXTHOP, sizeofSockaddrInet6, marshalNextHop, parseNextHop}, + ctlPathMTU: {sysIPV6_PATHMTU, sizeofIPv6Mtuinfo, marshalPathMTU, parsePathMTU}, + } + + sockOpts = map[int]*sockOpt{ + ssoTrafficClass: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_TCLASS, Len: 4}}, + ssoHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_UNICAST_HOPS, Len: 4}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_IF, Len: 4}}, + ssoMulticastHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_HOPS, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_LOOP, Len: 4}}, + ssoReceiveTrafficClass: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVTCLASS, Len: 4}}, + ssoReceiveHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVHOPLIMIT, Len: 4}}, + ssoReceivePacketInfo: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPKTINFO, Len: 4}}, + ssoReceivePathMTU: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_RECVPATHMTU, Len: 4}}, + ssoPathMTU: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_PATHMTU, Len: sizeofIPv6Mtuinfo}}, + ssoChecksum: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_CHECKSUM, Len: 4}}, + ssoICMPFilter: {Option: socket.Option{Level: iana.ProtocolIPv6ICMP, Name: sysICMP6_FILTER, Len: sizeofICMPv6Filter}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_JOIN_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_LEAVE_GROUP, Len: sizeofGroupReq}, typ: ssoTypeGroupReq}, + ssoJoinSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_JOIN_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoLeaveSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_LEAVE_SOURCE_GROUP, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoBlockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_BLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + ssoUnblockSourceGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysMCAST_UNBLOCK_SOURCE, Len: sizeofGroupSourceReq}, typ: ssoTypeGroupSourceReq}, + } +) + +func (sa *sockaddrInet6) setSockaddr(ip net.IP, i int) { + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(i) +} + +func (pi *inet6Pktinfo) setIfindex(i int) { + pi.Ifindex = uint32(i) +} + +func (mreq *ipv6Mreq) setIfindex(i int) { + mreq.Interface = uint32(i) +} + +func (gr *groupReq) setGroup(grp net.IP) { + sa := (*sockaddrInet6)(unsafe.Pointer(uintptr(unsafe.Pointer(gr)) + 4)) + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], grp) +} + +func (gsr *groupSourceReq) setSourceGroup(grp, src net.IP) { + sa := (*sockaddrInet6)(unsafe.Pointer(uintptr(unsafe.Pointer(gsr)) + 4)) + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], grp) + sa = (*sockaddrInet6)(unsafe.Pointer(uintptr(unsafe.Pointer(gsr)) + 260)) + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], src) +} diff --git a/vendor/golang.org/x/net/ipv6/sys_ssmreq.go b/vendor/golang.org/x/net/ipv6/sys_ssmreq.go new file mode 100644 index 00000000..add8ccc0 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_ssmreq.go @@ -0,0 +1,54 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd linux solaris + +package ipv6 + +import ( + "net" + "unsafe" + + "golang.org/x/net/internal/socket" +) + +var freebsd32o64 bool + +func (so *sockOpt) setGroupReq(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + var gr groupReq + if ifi != nil { + gr.Interface = uint32(ifi.Index) + } + gr.setGroup(grp) + var b []byte + if freebsd32o64 { + var d [sizeofGroupReq + 4]byte + s := (*[sizeofGroupReq]byte)(unsafe.Pointer(&gr)) + copy(d[:4], s[:4]) + copy(d[8:], s[4:]) + b = d[:] + } else { + b = (*[sizeofGroupReq]byte)(unsafe.Pointer(&gr))[:sizeofGroupReq] + } + return so.Set(c, b) +} + +func (so *sockOpt) setGroupSourceReq(c *socket.Conn, ifi *net.Interface, grp, src net.IP) error { + var gsr groupSourceReq + if ifi != nil { + gsr.Interface = uint32(ifi.Index) + } + gsr.setSourceGroup(grp, src) + var b []byte + if freebsd32o64 { + var d [sizeofGroupSourceReq + 4]byte + s := (*[sizeofGroupSourceReq]byte)(unsafe.Pointer(&gsr)) + copy(d[:4], s[:4]) + copy(d[8:], s[4:]) + b = d[:] + } else { + b = (*[sizeofGroupSourceReq]byte)(unsafe.Pointer(&gsr))[:sizeofGroupSourceReq] + } + return so.Set(c, b) +} diff --git a/vendor/golang.org/x/net/ipv6/sys_ssmreq_stub.go b/vendor/golang.org/x/net/ipv6/sys_ssmreq_stub.go new file mode 100644 index 00000000..581ee490 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_ssmreq_stub.go @@ -0,0 +1,21 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!freebsd,!linux,!solaris + +package ipv6 + +import ( + "net" + + "golang.org/x/net/internal/socket" +) + +func (so *sockOpt) setGroupReq(c *socket.Conn, ifi *net.Interface, grp net.IP) error { + return errOpNoSupport +} + +func (so *sockOpt) setGroupSourceReq(c *socket.Conn, ifi *net.Interface, grp, src net.IP) error { + return errOpNoSupport +} diff --git a/vendor/golang.org/x/net/ipv6/sys_stub.go b/vendor/golang.org/x/net/ipv6/sys_stub.go new file mode 100644 index 00000000..b845388e --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_stub.go @@ -0,0 +1,13 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package ipv6 + +var ( + ctlOpts = [ctlMax]ctlOpt{} + + sockOpts = map[int]*sockOpt{} +) diff --git a/vendor/golang.org/x/net/ipv6/sys_windows.go b/vendor/golang.org/x/net/ipv6/sys_windows.go new file mode 100644 index 00000000..fc36b018 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/sys_windows.go @@ -0,0 +1,75 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ipv6 + +import ( + "net" + "syscall" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/internal/socket" +) + +const ( + // See ws2tcpip.h. + sysIPV6_UNICAST_HOPS = 0x4 + sysIPV6_MULTICAST_IF = 0x9 + sysIPV6_MULTICAST_HOPS = 0xa + sysIPV6_MULTICAST_LOOP = 0xb + sysIPV6_JOIN_GROUP = 0xc + sysIPV6_LEAVE_GROUP = 0xd + sysIPV6_PKTINFO = 0x13 + + sizeofSockaddrInet6 = 0x1c + + sizeofIPv6Mreq = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofICMPv6Filter = 0 +) + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type icmpv6Filter struct { + // TODO(mikio): implement this +} + +var ( + ctlOpts = [ctlMax]ctlOpt{} + + sockOpts = map[int]*sockOpt{ + ssoHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_UNICAST_HOPS, Len: 4}}, + ssoMulticastInterface: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_IF, Len: 4}}, + ssoMulticastHopLimit: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_HOPS, Len: 4}}, + ssoMulticastLoopback: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_MULTICAST_LOOP, Len: 4}}, + ssoJoinGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_JOIN_GROUP, Len: sizeofIPv6Mreq}, typ: ssoTypeIPMreq}, + ssoLeaveGroup: {Option: socket.Option{Level: iana.ProtocolIPv6, Name: sysIPV6_LEAVE_GROUP, Len: sizeofIPv6Mreq}, typ: ssoTypeIPMreq}, + } +) + +func (sa *sockaddrInet6) setSockaddr(ip net.IP, i int) { + sa.Family = syscall.AF_INET6 + copy(sa.Addr[:], ip) + sa.Scope_id = uint32(i) +} + +func (mreq *ipv6Mreq) setIfindex(i int) { + mreq.Interface = uint32(i) +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_darwin.go b/vendor/golang.org/x/net/ipv6/zsys_darwin.go new file mode 100644 index 00000000..6aab1dfa --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_darwin.go @@ -0,0 +1,131 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_darwin.go + +package ipv6 + +const ( + sysIPV6_UNICAST_HOPS = 0x4 + sysIPV6_MULTICAST_IF = 0x9 + sysIPV6_MULTICAST_HOPS = 0xa + sysIPV6_MULTICAST_LOOP = 0xb + sysIPV6_JOIN_GROUP = 0xc + sysIPV6_LEAVE_GROUP = 0xd + + sysIPV6_PORTRANGE = 0xe + sysICMP6_FILTER = 0x12 + sysIPV6_2292PKTINFO = 0x13 + sysIPV6_2292HOPLIMIT = 0x14 + sysIPV6_2292NEXTHOP = 0x15 + sysIPV6_2292HOPOPTS = 0x16 + sysIPV6_2292DSTOPTS = 0x17 + sysIPV6_2292RTHDR = 0x18 + + sysIPV6_2292PKTOPTIONS = 0x19 + + sysIPV6_CHECKSUM = 0x1a + sysIPV6_V6ONLY = 0x1b + + sysIPV6_IPSEC_POLICY = 0x1c + + sysIPV6_RECVTCLASS = 0x23 + sysIPV6_TCLASS = 0x24 + + sysIPV6_RTHDRDSTOPTS = 0x39 + + sysIPV6_RECVPKTINFO = 0x3d + + sysIPV6_RECVHOPLIMIT = 0x25 + sysIPV6_RECVRTHDR = 0x26 + sysIPV6_RECVHOPOPTS = 0x27 + sysIPV6_RECVDSTOPTS = 0x28 + + sysIPV6_USE_MIN_MTU = 0x2a + sysIPV6_RECVPATHMTU = 0x2b + + sysIPV6_PATHMTU = 0x2c + + sysIPV6_PKTINFO = 0x2e + sysIPV6_HOPLIMIT = 0x2f + sysIPV6_NEXTHOP = 0x30 + sysIPV6_HOPOPTS = 0x31 + sysIPV6_DSTOPTS = 0x32 + sysIPV6_RTHDR = 0x33 + + sysIPV6_AUTOFLOWLABEL = 0x3b + + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_PREFER_TEMPADDR = 0x3f + + sysIPV6_MSFILTER = 0x4a + sysMCAST_JOIN_GROUP = 0x50 + sysMCAST_LEAVE_GROUP = 0x51 + sysMCAST_JOIN_SOURCE_GROUP = 0x52 + sysMCAST_LEAVE_SOURCE_GROUP = 0x53 + sysMCAST_BLOCK_SOURCE = 0x54 + sysMCAST_UNBLOCK_SOURCE = 0x55 + + sysIPV6_BOUND_IF = 0x7d + + sysIPV6_PORTRANGE_DEFAULT = 0x0 + sysIPV6_PORTRANGE_HIGH = 0x1 + sysIPV6_PORTRANGE_LOW = 0x2 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPv6Filter = 0x20 +) + +type sockaddrStorage struct { + Len uint8 + Family uint8 + X__ss_pad1 [6]int8 + X__ss_align int64 + X__ss_pad2 [112]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type icmpv6Filter struct { + Filt [8]uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [128]byte +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [128]byte + Pad_cgo_1 [128]byte +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_dragonfly.go b/vendor/golang.org/x/net/ipv6/zsys_dragonfly.go new file mode 100644 index 00000000..d2de804d --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_dragonfly.go @@ -0,0 +1,88 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_dragonfly.go + +package ipv6 + +const ( + sysIPV6_UNICAST_HOPS = 0x4 + sysIPV6_MULTICAST_IF = 0x9 + sysIPV6_MULTICAST_HOPS = 0xa + sysIPV6_MULTICAST_LOOP = 0xb + sysIPV6_JOIN_GROUP = 0xc + sysIPV6_LEAVE_GROUP = 0xd + sysIPV6_PORTRANGE = 0xe + sysICMP6_FILTER = 0x12 + + sysIPV6_CHECKSUM = 0x1a + sysIPV6_V6ONLY = 0x1b + + sysIPV6_IPSEC_POLICY = 0x1c + + sysIPV6_RTHDRDSTOPTS = 0x23 + sysIPV6_RECVPKTINFO = 0x24 + sysIPV6_RECVHOPLIMIT = 0x25 + sysIPV6_RECVRTHDR = 0x26 + sysIPV6_RECVHOPOPTS = 0x27 + sysIPV6_RECVDSTOPTS = 0x28 + + sysIPV6_USE_MIN_MTU = 0x2a + sysIPV6_RECVPATHMTU = 0x2b + + sysIPV6_PATHMTU = 0x2c + + sysIPV6_PKTINFO = 0x2e + sysIPV6_HOPLIMIT = 0x2f + sysIPV6_NEXTHOP = 0x30 + sysIPV6_HOPOPTS = 0x31 + sysIPV6_DSTOPTS = 0x32 + sysIPV6_RTHDR = 0x33 + + sysIPV6_RECVTCLASS = 0x39 + + sysIPV6_AUTOFLOWLABEL = 0x3b + + sysIPV6_TCLASS = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_PREFER_TEMPADDR = 0x3f + + sysIPV6_PORTRANGE_DEFAULT = 0x0 + sysIPV6_PORTRANGE_HIGH = 0x1 + sysIPV6_PORTRANGE_LOW = 0x2 + + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + + sizeofIPv6Mreq = 0x14 + + sizeofICMPv6Filter = 0x20 +) + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type icmpv6Filter struct { + Filt [8]uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_freebsd_386.go b/vendor/golang.org/x/net/ipv6/zsys_freebsd_386.go new file mode 100644 index 00000000..919e572d --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_freebsd_386.go @@ -0,0 +1,122 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package ipv6 + +const ( + sysIPV6_UNICAST_HOPS = 0x4 + sysIPV6_MULTICAST_IF = 0x9 + sysIPV6_MULTICAST_HOPS = 0xa + sysIPV6_MULTICAST_LOOP = 0xb + sysIPV6_JOIN_GROUP = 0xc + sysIPV6_LEAVE_GROUP = 0xd + sysIPV6_PORTRANGE = 0xe + sysICMP6_FILTER = 0x12 + + sysIPV6_CHECKSUM = 0x1a + sysIPV6_V6ONLY = 0x1b + + sysIPV6_IPSEC_POLICY = 0x1c + + sysIPV6_RTHDRDSTOPTS = 0x23 + + sysIPV6_RECVPKTINFO = 0x24 + sysIPV6_RECVHOPLIMIT = 0x25 + sysIPV6_RECVRTHDR = 0x26 + sysIPV6_RECVHOPOPTS = 0x27 + sysIPV6_RECVDSTOPTS = 0x28 + + sysIPV6_USE_MIN_MTU = 0x2a + sysIPV6_RECVPATHMTU = 0x2b + + sysIPV6_PATHMTU = 0x2c + + sysIPV6_PKTINFO = 0x2e + sysIPV6_HOPLIMIT = 0x2f + sysIPV6_NEXTHOP = 0x30 + sysIPV6_HOPOPTS = 0x31 + sysIPV6_DSTOPTS = 0x32 + sysIPV6_RTHDR = 0x33 + + sysIPV6_RECVTCLASS = 0x39 + + sysIPV6_AUTOFLOWLABEL = 0x3b + + sysIPV6_TCLASS = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_PREFER_TEMPADDR = 0x3f + + sysIPV6_BINDANY = 0x40 + + sysIPV6_MSFILTER = 0x4a + + sysMCAST_JOIN_GROUP = 0x50 + sysMCAST_LEAVE_GROUP = 0x51 + sysMCAST_JOIN_SOURCE_GROUP = 0x52 + sysMCAST_LEAVE_SOURCE_GROUP = 0x53 + sysMCAST_BLOCK_SOURCE = 0x54 + sysMCAST_UNBLOCK_SOURCE = 0x55 + + sysIPV6_PORTRANGE_DEFAULT = 0x0 + sysIPV6_PORTRANGE_HIGH = 0x1 + sysIPV6_PORTRANGE_LOW = 0x2 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPv6Filter = 0x20 +) + +type sockaddrStorage struct { + Len uint8 + Family uint8 + X__ss_pad1 [6]int8 + X__ss_align int64 + X__ss_pad2 [112]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type groupReq struct { + Interface uint32 + Group sockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group sockaddrStorage + Source sockaddrStorage +} + +type icmpv6Filter struct { + Filt [8]uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_freebsd_amd64.go b/vendor/golang.org/x/net/ipv6/zsys_freebsd_amd64.go new file mode 100644 index 00000000..cb8141f9 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_freebsd_amd64.go @@ -0,0 +1,124 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package ipv6 + +const ( + sysIPV6_UNICAST_HOPS = 0x4 + sysIPV6_MULTICAST_IF = 0x9 + sysIPV6_MULTICAST_HOPS = 0xa + sysIPV6_MULTICAST_LOOP = 0xb + sysIPV6_JOIN_GROUP = 0xc + sysIPV6_LEAVE_GROUP = 0xd + sysIPV6_PORTRANGE = 0xe + sysICMP6_FILTER = 0x12 + + sysIPV6_CHECKSUM = 0x1a + sysIPV6_V6ONLY = 0x1b + + sysIPV6_IPSEC_POLICY = 0x1c + + sysIPV6_RTHDRDSTOPTS = 0x23 + + sysIPV6_RECVPKTINFO = 0x24 + sysIPV6_RECVHOPLIMIT = 0x25 + sysIPV6_RECVRTHDR = 0x26 + sysIPV6_RECVHOPOPTS = 0x27 + sysIPV6_RECVDSTOPTS = 0x28 + + sysIPV6_USE_MIN_MTU = 0x2a + sysIPV6_RECVPATHMTU = 0x2b + + sysIPV6_PATHMTU = 0x2c + + sysIPV6_PKTINFO = 0x2e + sysIPV6_HOPLIMIT = 0x2f + sysIPV6_NEXTHOP = 0x30 + sysIPV6_HOPOPTS = 0x31 + sysIPV6_DSTOPTS = 0x32 + sysIPV6_RTHDR = 0x33 + + sysIPV6_RECVTCLASS = 0x39 + + sysIPV6_AUTOFLOWLABEL = 0x3b + + sysIPV6_TCLASS = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_PREFER_TEMPADDR = 0x3f + + sysIPV6_BINDANY = 0x40 + + sysIPV6_MSFILTER = 0x4a + + sysMCAST_JOIN_GROUP = 0x50 + sysMCAST_LEAVE_GROUP = 0x51 + sysMCAST_JOIN_SOURCE_GROUP = 0x52 + sysMCAST_LEAVE_SOURCE_GROUP = 0x53 + sysMCAST_BLOCK_SOURCE = 0x54 + sysMCAST_UNBLOCK_SOURCE = 0x55 + + sysIPV6_PORTRANGE_DEFAULT = 0x0 + sysIPV6_PORTRANGE_HIGH = 0x1 + sysIPV6_PORTRANGE_LOW = 0x2 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 +) + +type sockaddrStorage struct { + Len uint8 + Family uint8 + X__ss_pad1 [6]int8 + X__ss_align int64 + X__ss_pad2 [112]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group sockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group sockaddrStorage + Source sockaddrStorage +} + +type icmpv6Filter struct { + Filt [8]uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_freebsd_arm.go b/vendor/golang.org/x/net/ipv6/zsys_freebsd_arm.go new file mode 100644 index 00000000..cb8141f9 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_freebsd_arm.go @@ -0,0 +1,124 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_freebsd.go + +package ipv6 + +const ( + sysIPV6_UNICAST_HOPS = 0x4 + sysIPV6_MULTICAST_IF = 0x9 + sysIPV6_MULTICAST_HOPS = 0xa + sysIPV6_MULTICAST_LOOP = 0xb + sysIPV6_JOIN_GROUP = 0xc + sysIPV6_LEAVE_GROUP = 0xd + sysIPV6_PORTRANGE = 0xe + sysICMP6_FILTER = 0x12 + + sysIPV6_CHECKSUM = 0x1a + sysIPV6_V6ONLY = 0x1b + + sysIPV6_IPSEC_POLICY = 0x1c + + sysIPV6_RTHDRDSTOPTS = 0x23 + + sysIPV6_RECVPKTINFO = 0x24 + sysIPV6_RECVHOPLIMIT = 0x25 + sysIPV6_RECVRTHDR = 0x26 + sysIPV6_RECVHOPOPTS = 0x27 + sysIPV6_RECVDSTOPTS = 0x28 + + sysIPV6_USE_MIN_MTU = 0x2a + sysIPV6_RECVPATHMTU = 0x2b + + sysIPV6_PATHMTU = 0x2c + + sysIPV6_PKTINFO = 0x2e + sysIPV6_HOPLIMIT = 0x2f + sysIPV6_NEXTHOP = 0x30 + sysIPV6_HOPOPTS = 0x31 + sysIPV6_DSTOPTS = 0x32 + sysIPV6_RTHDR = 0x33 + + sysIPV6_RECVTCLASS = 0x39 + + sysIPV6_AUTOFLOWLABEL = 0x3b + + sysIPV6_TCLASS = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_PREFER_TEMPADDR = 0x3f + + sysIPV6_BINDANY = 0x40 + + sysIPV6_MSFILTER = 0x4a + + sysMCAST_JOIN_GROUP = 0x50 + sysMCAST_LEAVE_GROUP = 0x51 + sysMCAST_JOIN_SOURCE_GROUP = 0x52 + sysMCAST_LEAVE_SOURCE_GROUP = 0x53 + sysMCAST_BLOCK_SOURCE = 0x54 + sysMCAST_UNBLOCK_SOURCE = 0x55 + + sysIPV6_PORTRANGE_DEFAULT = 0x0 + sysIPV6_PORTRANGE_HIGH = 0x1 + sysIPV6_PORTRANGE_LOW = 0x2 + + sizeofSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 +) + +type sockaddrStorage struct { + Len uint8 + Family uint8 + X__ss_pad1 [6]int8 + X__ss_align int64 + X__ss_pad2 [112]int8 +} + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group sockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group sockaddrStorage + Source sockaddrStorage +} + +type icmpv6Filter struct { + Filt [8]uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_386.go b/vendor/golang.org/x/net/ipv6/zsys_linux_386.go new file mode 100644 index 00000000..73aa8c6d --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_386.go @@ -0,0 +1,170 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_amd64.go b/vendor/golang.org/x/net/ipv6/zsys_linux_amd64.go new file mode 100644 index 00000000..b64f0157 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_amd64.go @@ -0,0 +1,172 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_arm.go b/vendor/golang.org/x/net/ipv6/zsys_linux_arm.go new file mode 100644 index 00000000..73aa8c6d --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_arm.go @@ -0,0 +1,170 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_arm64.go b/vendor/golang.org/x/net/ipv6/zsys_linux_arm64.go new file mode 100644 index 00000000..b64f0157 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_arm64.go @@ -0,0 +1,172 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_mips.go b/vendor/golang.org/x/net/ipv6/zsys_linux_mips.go new file mode 100644 index 00000000..73aa8c6d --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_mips.go @@ -0,0 +1,170 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_mips64.go b/vendor/golang.org/x/net/ipv6/zsys_linux_mips64.go new file mode 100644 index 00000000..b64f0157 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_mips64.go @@ -0,0 +1,172 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_mips64le.go b/vendor/golang.org/x/net/ipv6/zsys_linux_mips64le.go new file mode 100644 index 00000000..b64f0157 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_mips64le.go @@ -0,0 +1,172 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_mipsle.go b/vendor/golang.org/x/net/ipv6/zsys_linux_mipsle.go new file mode 100644 index 00000000..73aa8c6d --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_mipsle.go @@ -0,0 +1,170 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_ppc.go b/vendor/golang.org/x/net/ipv6/zsys_linux_ppc.go new file mode 100644 index 00000000..c9bf6a87 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_ppc.go @@ -0,0 +1,170 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x84 + sizeofGroupSourceReq = 0x104 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x8 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]uint8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [2]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_ppc64.go b/vendor/golang.org/x/net/ipv6/zsys_linux_ppc64.go new file mode 100644 index 00000000..b64f0157 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_ppc64.go @@ -0,0 +1,172 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_ppc64le.go b/vendor/golang.org/x/net/ipv6/zsys_linux_ppc64le.go new file mode 100644 index 00000000..b64f0157 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_ppc64le.go @@ -0,0 +1,172 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_linux_s390x.go b/vendor/golang.org/x/net/ipv6/zsys_linux_s390x.go new file mode 100644 index 00000000..b64f0157 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_linux_s390x.go @@ -0,0 +1,172 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_linux.go + +package ipv6 + +const ( + sysIPV6_ADDRFORM = 0x1 + sysIPV6_2292PKTINFO = 0x2 + sysIPV6_2292HOPOPTS = 0x3 + sysIPV6_2292DSTOPTS = 0x4 + sysIPV6_2292RTHDR = 0x5 + sysIPV6_2292PKTOPTIONS = 0x6 + sysIPV6_CHECKSUM = 0x7 + sysIPV6_2292HOPLIMIT = 0x8 + sysIPV6_NEXTHOP = 0x9 + sysIPV6_FLOWINFO = 0xb + + sysIPV6_UNICAST_HOPS = 0x10 + sysIPV6_MULTICAST_IF = 0x11 + sysIPV6_MULTICAST_HOPS = 0x12 + sysIPV6_MULTICAST_LOOP = 0x13 + sysIPV6_ADD_MEMBERSHIP = 0x14 + sysIPV6_DROP_MEMBERSHIP = 0x15 + sysMCAST_JOIN_GROUP = 0x2a + sysMCAST_LEAVE_GROUP = 0x2d + sysMCAST_JOIN_SOURCE_GROUP = 0x2e + sysMCAST_LEAVE_SOURCE_GROUP = 0x2f + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_MSFILTER = 0x30 + sysIPV6_ROUTER_ALERT = 0x16 + sysIPV6_MTU_DISCOVER = 0x17 + sysIPV6_MTU = 0x18 + sysIPV6_RECVERR = 0x19 + sysIPV6_V6ONLY = 0x1a + sysIPV6_JOIN_ANYCAST = 0x1b + sysIPV6_LEAVE_ANYCAST = 0x1c + + sysIPV6_FLOWLABEL_MGR = 0x20 + sysIPV6_FLOWINFO_SEND = 0x21 + + sysIPV6_IPSEC_POLICY = 0x22 + sysIPV6_XFRM_POLICY = 0x23 + + sysIPV6_RECVPKTINFO = 0x31 + sysIPV6_PKTINFO = 0x32 + sysIPV6_RECVHOPLIMIT = 0x33 + sysIPV6_HOPLIMIT = 0x34 + sysIPV6_RECVHOPOPTS = 0x35 + sysIPV6_HOPOPTS = 0x36 + sysIPV6_RTHDRDSTOPTS = 0x37 + sysIPV6_RECVRTHDR = 0x38 + sysIPV6_RTHDR = 0x39 + sysIPV6_RECVDSTOPTS = 0x3a + sysIPV6_DSTOPTS = 0x3b + sysIPV6_RECVPATHMTU = 0x3c + sysIPV6_PATHMTU = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_RECVTCLASS = 0x42 + sysIPV6_TCLASS = 0x43 + + sysIPV6_ADDR_PREFERENCES = 0x48 + + sysIPV6_PREFER_SRC_TMP = 0x1 + sysIPV6_PREFER_SRC_PUBLIC = 0x2 + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = 0x100 + sysIPV6_PREFER_SRC_COA = 0x4 + sysIPV6_PREFER_SRC_HOME = 0x400 + sysIPV6_PREFER_SRC_CGA = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x800 + + sysIPV6_MINHOPCOUNT = 0x49 + + sysIPV6_ORIGDSTADDR = 0x4a + sysIPV6_RECVORIGDSTADDR = 0x4a + sysIPV6_TRANSPARENT = 0x4b + sysIPV6_UNICAST_IF = 0x4c + + sysICMPV6_FILTER = 0x1 + + sysICMPV6_FILTER_BLOCK = 0x1 + sysICMPV6_FILTER_PASS = 0x2 + sysICMPV6_FILTER_BLOCKOTHERS = 0x3 + sysICMPV6_FILTER_PASSONLY = 0x4 + + sysSOL_SOCKET = 0x1 + sysSO_ATTACH_FILTER = 0x1a + + sizeofKernelSockaddrStorage = 0x80 + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + sizeofIPv6FlowlabelReq = 0x20 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x88 + sizeofGroupSourceReq = 0x108 + + sizeofICMPv6Filter = 0x20 + + sizeofSockFprog = 0x10 +) + +type kernelSockaddrStorage struct { + Family uint16 + X__data [126]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex int32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6FlowlabelReq struct { + Dst [16]byte /* in6_addr */ + Label uint32 + Action uint8 + Share uint8 + Flags uint16 + Expires uint16 + Linger uint16 + X__flr_pad uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Ifindex int32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [4]byte + Group kernelSockaddrStorage + Source kernelSockaddrStorage +} + +type icmpv6Filter struct { + Data [8]uint32 +} + +type sockFProg struct { + Len uint16 + Pad_cgo_0 [6]byte + Filter *sockFilter +} + +type sockFilter struct { + Code uint16 + Jt uint8 + Jf uint8 + K uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_netbsd.go b/vendor/golang.org/x/net/ipv6/zsys_netbsd.go new file mode 100644 index 00000000..bcada13b --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_netbsd.go @@ -0,0 +1,84 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_netbsd.go + +package ipv6 + +const ( + sysIPV6_UNICAST_HOPS = 0x4 + sysIPV6_MULTICAST_IF = 0x9 + sysIPV6_MULTICAST_HOPS = 0xa + sysIPV6_MULTICAST_LOOP = 0xb + sysIPV6_JOIN_GROUP = 0xc + sysIPV6_LEAVE_GROUP = 0xd + sysIPV6_PORTRANGE = 0xe + sysICMP6_FILTER = 0x12 + + sysIPV6_CHECKSUM = 0x1a + sysIPV6_V6ONLY = 0x1b + + sysIPV6_IPSEC_POLICY = 0x1c + + sysIPV6_RTHDRDSTOPTS = 0x23 + + sysIPV6_RECVPKTINFO = 0x24 + sysIPV6_RECVHOPLIMIT = 0x25 + sysIPV6_RECVRTHDR = 0x26 + sysIPV6_RECVHOPOPTS = 0x27 + sysIPV6_RECVDSTOPTS = 0x28 + + sysIPV6_USE_MIN_MTU = 0x2a + sysIPV6_RECVPATHMTU = 0x2b + sysIPV6_PATHMTU = 0x2c + + sysIPV6_PKTINFO = 0x2e + sysIPV6_HOPLIMIT = 0x2f + sysIPV6_NEXTHOP = 0x30 + sysIPV6_HOPOPTS = 0x31 + sysIPV6_DSTOPTS = 0x32 + sysIPV6_RTHDR = 0x33 + + sysIPV6_RECVTCLASS = 0x39 + + sysIPV6_TCLASS = 0x3d + sysIPV6_DONTFRAG = 0x3e + + sysIPV6_PORTRANGE_DEFAULT = 0x0 + sysIPV6_PORTRANGE_HIGH = 0x1 + sysIPV6_PORTRANGE_LOW = 0x2 + + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + + sizeofIPv6Mreq = 0x14 + + sizeofICMPv6Filter = 0x20 +) + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type icmpv6Filter struct { + Filt [8]uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_openbsd.go b/vendor/golang.org/x/net/ipv6/zsys_openbsd.go new file mode 100644 index 00000000..86cf3c63 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_openbsd.go @@ -0,0 +1,93 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_openbsd.go + +package ipv6 + +const ( + sysIPV6_UNICAST_HOPS = 0x4 + sysIPV6_MULTICAST_IF = 0x9 + sysIPV6_MULTICAST_HOPS = 0xa + sysIPV6_MULTICAST_LOOP = 0xb + sysIPV6_JOIN_GROUP = 0xc + sysIPV6_LEAVE_GROUP = 0xd + sysIPV6_PORTRANGE = 0xe + sysICMP6_FILTER = 0x12 + + sysIPV6_CHECKSUM = 0x1a + sysIPV6_V6ONLY = 0x1b + + sysIPV6_RTHDRDSTOPTS = 0x23 + + sysIPV6_RECVPKTINFO = 0x24 + sysIPV6_RECVHOPLIMIT = 0x25 + sysIPV6_RECVRTHDR = 0x26 + sysIPV6_RECVHOPOPTS = 0x27 + sysIPV6_RECVDSTOPTS = 0x28 + + sysIPV6_USE_MIN_MTU = 0x2a + sysIPV6_RECVPATHMTU = 0x2b + + sysIPV6_PATHMTU = 0x2c + + sysIPV6_PKTINFO = 0x2e + sysIPV6_HOPLIMIT = 0x2f + sysIPV6_NEXTHOP = 0x30 + sysIPV6_HOPOPTS = 0x31 + sysIPV6_DSTOPTS = 0x32 + sysIPV6_RTHDR = 0x33 + + sysIPV6_AUTH_LEVEL = 0x35 + sysIPV6_ESP_TRANS_LEVEL = 0x36 + sysIPV6_ESP_NETWORK_LEVEL = 0x37 + sysIPSEC6_OUTSA = 0x38 + sysIPV6_RECVTCLASS = 0x39 + + sysIPV6_AUTOFLOWLABEL = 0x3b + sysIPV6_IPCOMP_LEVEL = 0x3c + + sysIPV6_TCLASS = 0x3d + sysIPV6_DONTFRAG = 0x3e + sysIPV6_PIPEX = 0x3f + + sysIPV6_RTABLE = 0x1021 + + sysIPV6_PORTRANGE_DEFAULT = 0x0 + sysIPV6_PORTRANGE_HIGH = 0x1 + sysIPV6_PORTRANGE_LOW = 0x2 + + sizeofSockaddrInet6 = 0x1c + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x20 + + sizeofIPv6Mreq = 0x14 + + sizeofICMPv6Filter = 0x20 +) + +type sockaddrInet6 struct { + Len uint8 + Family uint8 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type icmpv6Filter struct { + Filt [8]uint32 +} diff --git a/vendor/golang.org/x/net/ipv6/zsys_solaris.go b/vendor/golang.org/x/net/ipv6/zsys_solaris.go new file mode 100644 index 00000000..cf1837dd --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/zsys_solaris.go @@ -0,0 +1,131 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs defs_solaris.go + +package ipv6 + +const ( + sysIPV6_UNICAST_HOPS = 0x5 + sysIPV6_MULTICAST_IF = 0x6 + sysIPV6_MULTICAST_HOPS = 0x7 + sysIPV6_MULTICAST_LOOP = 0x8 + sysIPV6_JOIN_GROUP = 0x9 + sysIPV6_LEAVE_GROUP = 0xa + + sysIPV6_PKTINFO = 0xb + + sysIPV6_HOPLIMIT = 0xc + sysIPV6_NEXTHOP = 0xd + sysIPV6_HOPOPTS = 0xe + sysIPV6_DSTOPTS = 0xf + + sysIPV6_RTHDR = 0x10 + sysIPV6_RTHDRDSTOPTS = 0x11 + + sysIPV6_RECVPKTINFO = 0x12 + sysIPV6_RECVHOPLIMIT = 0x13 + sysIPV6_RECVHOPOPTS = 0x14 + + sysIPV6_RECVRTHDR = 0x16 + + sysIPV6_RECVRTHDRDSTOPTS = 0x17 + + sysIPV6_CHECKSUM = 0x18 + sysIPV6_RECVTCLASS = 0x19 + sysIPV6_USE_MIN_MTU = 0x20 + sysIPV6_DONTFRAG = 0x21 + sysIPV6_SEC_OPT = 0x22 + sysIPV6_SRC_PREFERENCES = 0x23 + sysIPV6_RECVPATHMTU = 0x24 + sysIPV6_PATHMTU = 0x25 + sysIPV6_TCLASS = 0x26 + sysIPV6_V6ONLY = 0x27 + + sysIPV6_RECVDSTOPTS = 0x28 + + sysMCAST_JOIN_GROUP = 0x29 + sysMCAST_LEAVE_GROUP = 0x2a + sysMCAST_BLOCK_SOURCE = 0x2b + sysMCAST_UNBLOCK_SOURCE = 0x2c + sysMCAST_JOIN_SOURCE_GROUP = 0x2d + sysMCAST_LEAVE_SOURCE_GROUP = 0x2e + + sysIPV6_PREFER_SRC_HOME = 0x1 + sysIPV6_PREFER_SRC_COA = 0x2 + sysIPV6_PREFER_SRC_PUBLIC = 0x4 + sysIPV6_PREFER_SRC_TMP = 0x8 + sysIPV6_PREFER_SRC_NONCGA = 0x10 + sysIPV6_PREFER_SRC_CGA = 0x20 + + sysIPV6_PREFER_SRC_MIPMASK = 0x3 + sysIPV6_PREFER_SRC_MIPDEFAULT = 0x1 + sysIPV6_PREFER_SRC_TMPMASK = 0xc + sysIPV6_PREFER_SRC_TMPDEFAULT = 0x4 + sysIPV6_PREFER_SRC_CGAMASK = 0x30 + sysIPV6_PREFER_SRC_CGADEFAULT = 0x10 + + sysIPV6_PREFER_SRC_MASK = 0x3f + + sysIPV6_PREFER_SRC_DEFAULT = 0x15 + + sysIPV6_BOUND_IF = 0x41 + sysIPV6_UNSPEC_SRC = 0x42 + + sysICMP6_FILTER = 0x1 + + sizeofSockaddrStorage = 0x100 + sizeofSockaddrInet6 = 0x20 + sizeofInet6Pktinfo = 0x14 + sizeofIPv6Mtuinfo = 0x24 + + sizeofIPv6Mreq = 0x14 + sizeofGroupReq = 0x104 + sizeofGroupSourceReq = 0x204 + + sizeofICMPv6Filter = 0x20 +) + +type sockaddrStorage struct { + Family uint16 + X_ss_pad1 [6]int8 + X_ss_align float64 + X_ss_pad2 [240]int8 +} + +type sockaddrInet6 struct { + Family uint16 + Port uint16 + Flowinfo uint32 + Addr [16]byte /* in6_addr */ + Scope_id uint32 + X__sin6_src_id uint32 +} + +type inet6Pktinfo struct { + Addr [16]byte /* in6_addr */ + Ifindex uint32 +} + +type ipv6Mtuinfo struct { + Addr sockaddrInet6 + Mtu uint32 +} + +type ipv6Mreq struct { + Multiaddr [16]byte /* in6_addr */ + Interface uint32 +} + +type groupReq struct { + Interface uint32 + Pad_cgo_0 [256]byte +} + +type groupSourceReq struct { + Interface uint32 + Pad_cgo_0 [256]byte + Pad_cgo_1 [256]byte +} + +type icmpv6Filter struct { + X__icmp6_filt [8]uint32 +} diff --git a/vendor/golang.org/x/net/publicsuffix/gen.go b/vendor/golang.org/x/net/publicsuffix/gen.go deleted file mode 100644 index a2d49952..00000000 --- a/vendor/golang.org/x/net/publicsuffix/gen.go +++ /dev/null @@ -1,713 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build ignore - -package main - -// This program generates table.go and table_test.go based on the authoritative -// public suffix list at https://publicsuffix.org/list/effective_tld_names.dat -// -// The version is derived from -// https://api.github.com/repos/publicsuffix/list/commits?path=public_suffix_list.dat -// and a human-readable form is at -// https://github.com/publicsuffix/list/commits/master/public_suffix_list.dat -// -// To fetch a particular git revision, such as 5c70ccd250, pass -// -url "https://raw.githubusercontent.com/publicsuffix/list/5c70ccd250/public_suffix_list.dat" -// and -version "an explicit version string". - -import ( - "bufio" - "bytes" - "flag" - "fmt" - "go/format" - "io" - "io/ioutil" - "net/http" - "os" - "regexp" - "sort" - "strings" - - "golang.org/x/net/idna" -) - -const ( - // These sum of these four values must be no greater than 32. - nodesBitsChildren = 9 - nodesBitsICANN = 1 - nodesBitsTextOffset = 15 - nodesBitsTextLength = 6 - - // These sum of these four values must be no greater than 32. - childrenBitsWildcard = 1 - childrenBitsNodeType = 2 - childrenBitsHi = 14 - childrenBitsLo = 14 -) - -var ( - maxChildren int - maxTextOffset int - maxTextLength int - maxHi uint32 - maxLo uint32 -) - -func max(a, b int) int { - if a < b { - return b - } - return a -} - -func u32max(a, b uint32) uint32 { - if a < b { - return b - } - return a -} - -const ( - nodeTypeNormal = 0 - nodeTypeException = 1 - nodeTypeParentOnly = 2 - numNodeType = 3 -) - -func nodeTypeStr(n int) string { - switch n { - case nodeTypeNormal: - return "+" - case nodeTypeException: - return "!" - case nodeTypeParentOnly: - return "o" - } - panic("unreachable") -} - -const ( - defaultURL = "https://publicsuffix.org/list/effective_tld_names.dat" - gitCommitURL = "https://api.github.com/repos/publicsuffix/list/commits?path=public_suffix_list.dat" -) - -var ( - labelEncoding = map[string]uint32{} - labelsList = []string{} - labelsMap = map[string]bool{} - rules = []string{} - - // validSuffixRE is used to check that the entries in the public suffix - // list are in canonical form (after Punycode encoding). Specifically, - // capital letters are not allowed. - validSuffixRE = regexp.MustCompile(`^[a-z0-9_\!\*\-\.]+$`) - - shaRE = regexp.MustCompile(`"sha":"([^"]+)"`) - dateRE = regexp.MustCompile(`"committer":{[^{]+"date":"([^"]+)"`) - - comments = flag.Bool("comments", false, "generate table.go comments, for debugging") - subset = flag.Bool("subset", false, "generate only a subset of the full table, for debugging") - url = flag.String("url", defaultURL, "URL of the publicsuffix.org list. If empty, stdin is read instead") - v = flag.Bool("v", false, "verbose output (to stderr)") - version = flag.String("version", "", "the effective_tld_names.dat version") -) - -func main() { - if err := main1(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func main1() error { - flag.Parse() - if nodesBitsTextLength+nodesBitsTextOffset+nodesBitsICANN+nodesBitsChildren > 32 { - return fmt.Errorf("not enough bits to encode the nodes table") - } - if childrenBitsLo+childrenBitsHi+childrenBitsNodeType+childrenBitsWildcard > 32 { - return fmt.Errorf("not enough bits to encode the children table") - } - if *version == "" { - if *url != defaultURL { - return fmt.Errorf("-version was not specified, and the -url is not the default one") - } - sha, date, err := gitCommit() - if err != nil { - return err - } - *version = fmt.Sprintf("publicsuffix.org's public_suffix_list.dat, git revision %s (%s)", sha, date) - } - var r io.Reader = os.Stdin - if *url != "" { - res, err := http.Get(*url) - if err != nil { - return err - } - if res.StatusCode != http.StatusOK { - return fmt.Errorf("bad GET status for %s: %d", *url, res.Status) - } - r = res.Body - defer res.Body.Close() - } - - var root node - icann := false - br := bufio.NewReader(r) - for { - s, err := br.ReadString('\n') - if err != nil { - if err == io.EOF { - break - } - return err - } - s = strings.TrimSpace(s) - if strings.Contains(s, "BEGIN ICANN DOMAINS") { - icann = true - continue - } - if strings.Contains(s, "END ICANN DOMAINS") { - icann = false - continue - } - if s == "" || strings.HasPrefix(s, "//") { - continue - } - s, err = idna.ToASCII(s) - if err != nil { - return err - } - if !validSuffixRE.MatchString(s) { - return fmt.Errorf("bad publicsuffix.org list data: %q", s) - } - - if *subset { - switch { - case s == "ac.jp" || strings.HasSuffix(s, ".ac.jp"): - case s == "ak.us" || strings.HasSuffix(s, ".ak.us"): - case s == "ao" || strings.HasSuffix(s, ".ao"): - case s == "ar" || strings.HasSuffix(s, ".ar"): - case s == "arpa" || strings.HasSuffix(s, ".arpa"): - case s == "cy" || strings.HasSuffix(s, ".cy"): - case s == "dyndns.org" || strings.HasSuffix(s, ".dyndns.org"): - case s == "jp": - case s == "kobe.jp" || strings.HasSuffix(s, ".kobe.jp"): - case s == "kyoto.jp" || strings.HasSuffix(s, ".kyoto.jp"): - case s == "om" || strings.HasSuffix(s, ".om"): - case s == "uk" || strings.HasSuffix(s, ".uk"): - case s == "uk.com" || strings.HasSuffix(s, ".uk.com"): - case s == "tw" || strings.HasSuffix(s, ".tw"): - case s == "zw" || strings.HasSuffix(s, ".zw"): - case s == "xn--p1ai" || strings.HasSuffix(s, ".xn--p1ai"): - // xn--p1ai is Russian-Cyrillic "рф". - default: - continue - } - } - - rules = append(rules, s) - - nt, wildcard := nodeTypeNormal, false - switch { - case strings.HasPrefix(s, "*."): - s, nt = s[2:], nodeTypeParentOnly - wildcard = true - case strings.HasPrefix(s, "!"): - s, nt = s[1:], nodeTypeException - } - labels := strings.Split(s, ".") - for n, i := &root, len(labels)-1; i >= 0; i-- { - label := labels[i] - n = n.child(label) - if i == 0 { - if nt != nodeTypeParentOnly && n.nodeType == nodeTypeParentOnly { - n.nodeType = nt - } - n.icann = n.icann && icann - n.wildcard = n.wildcard || wildcard - } - labelsMap[label] = true - } - } - labelsList = make([]string, 0, len(labelsMap)) - for label := range labelsMap { - labelsList = append(labelsList, label) - } - sort.Strings(labelsList) - - if err := generate(printReal, &root, "table.go"); err != nil { - return err - } - if err := generate(printTest, &root, "table_test.go"); err != nil { - return err - } - return nil -} - -func generate(p func(io.Writer, *node) error, root *node, filename string) error { - buf := new(bytes.Buffer) - if err := p(buf, root); err != nil { - return err - } - b, err := format.Source(buf.Bytes()) - if err != nil { - return err - } - return ioutil.WriteFile(filename, b, 0644) -} - -func gitCommit() (sha, date string, retErr error) { - res, err := http.Get(gitCommitURL) - if err != nil { - return "", "", err - } - if res.StatusCode != http.StatusOK { - return "", "", fmt.Errorf("bad GET status for %s: %d", gitCommitURL, res.Status) - } - defer res.Body.Close() - b, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", "", err - } - if m := shaRE.FindSubmatch(b); m != nil { - sha = string(m[1]) - } - if m := dateRE.FindSubmatch(b); m != nil { - date = string(m[1]) - } - if sha == "" || date == "" { - retErr = fmt.Errorf("could not find commit SHA and date in %s", gitCommitURL) - } - return sha, date, retErr -} - -func printTest(w io.Writer, n *node) error { - fmt.Fprintf(w, "// generated by go run gen.go; DO NOT EDIT\n\n") - fmt.Fprintf(w, "package publicsuffix\n\nvar rules = [...]string{\n") - for _, rule := range rules { - fmt.Fprintf(w, "%q,\n", rule) - } - fmt.Fprintf(w, "}\n\nvar nodeLabels = [...]string{\n") - if err := n.walk(w, printNodeLabel); err != nil { - return err - } - fmt.Fprintf(w, "}\n") - return nil -} - -func printReal(w io.Writer, n *node) error { - const header = `// generated by go run gen.go; DO NOT EDIT - -package publicsuffix - -const version = %q - -const ( - nodesBitsChildren = %d - nodesBitsICANN = %d - nodesBitsTextOffset = %d - nodesBitsTextLength = %d - - childrenBitsWildcard = %d - childrenBitsNodeType = %d - childrenBitsHi = %d - childrenBitsLo = %d -) - -const ( - nodeTypeNormal = %d - nodeTypeException = %d - nodeTypeParentOnly = %d -) - -// numTLD is the number of top level domains. -const numTLD = %d - -` - fmt.Fprintf(w, header, *version, - nodesBitsChildren, nodesBitsICANN, nodesBitsTextOffset, nodesBitsTextLength, - childrenBitsWildcard, childrenBitsNodeType, childrenBitsHi, childrenBitsLo, - nodeTypeNormal, nodeTypeException, nodeTypeParentOnly, len(n.children)) - - text := combineText(labelsList) - if text == "" { - return fmt.Errorf("internal error: makeText returned no text") - } - for _, label := range labelsList { - offset, length := strings.Index(text, label), len(label) - if offset < 0 { - return fmt.Errorf("internal error: could not find %q in text %q", label, text) - } - maxTextOffset, maxTextLength = max(maxTextOffset, offset), max(maxTextLength, length) - if offset >= 1<= 1< 64 { - n, plus = 64, " +" - } - fmt.Fprintf(w, "%q%s\n", text[:n], plus) - text = text[n:] - } - - if err := n.walk(w, assignIndexes); err != nil { - return err - } - - fmt.Fprintf(w, ` - -// nodes is the list of nodes. Each node is represented as a uint32, which -// encodes the node's children, wildcard bit and node type (as an index into -// the children array), ICANN bit and text. -// -// If the table was generated with the -comments flag, there is a //-comment -// after each node's data. In it is the nodes-array indexes of the children, -// formatted as (n0x1234-n0x1256), with * denoting the wildcard bit. The -// nodeType is printed as + for normal, ! for exception, and o for parent-only -// nodes that have children but don't match a domain label in their own right. -// An I denotes an ICANN domain. -// -// The layout within the uint32, from MSB to LSB, is: -// [%2d bits] unused -// [%2d bits] children index -// [%2d bits] ICANN bit -// [%2d bits] text index -// [%2d bits] text length -var nodes = [...]uint32{ -`, - 32-nodesBitsChildren-nodesBitsICANN-nodesBitsTextOffset-nodesBitsTextLength, - nodesBitsChildren, nodesBitsICANN, nodesBitsTextOffset, nodesBitsTextLength) - if err := n.walk(w, printNode); err != nil { - return err - } - fmt.Fprintf(w, `} - -// children is the list of nodes' children, the parent's wildcard bit and the -// parent's node type. If a node has no children then their children index -// will be in the range [0, 6), depending on the wildcard bit and node type. -// -// The layout within the uint32, from MSB to LSB, is: -// [%2d bits] unused -// [%2d bits] wildcard bit -// [%2d bits] node type -// [%2d bits] high nodes index (exclusive) of children -// [%2d bits] low nodes index (inclusive) of children -var children=[...]uint32{ -`, - 32-childrenBitsWildcard-childrenBitsNodeType-childrenBitsHi-childrenBitsLo, - childrenBitsWildcard, childrenBitsNodeType, childrenBitsHi, childrenBitsLo) - for i, c := range childrenEncoding { - s := "---------------" - lo := c & (1<> childrenBitsLo) & (1<>(childrenBitsLo+childrenBitsHi)) & (1<>(childrenBitsLo+childrenBitsHi+childrenBitsNodeType) != 0 - if *comments { - fmt.Fprintf(w, "0x%08x, // c0x%04x (%s)%s %s\n", - c, i, s, wildcardStr(wildcard), nodeTypeStr(nodeType)) - } else { - fmt.Fprintf(w, "0x%x,\n", c) - } - } - fmt.Fprintf(w, "}\n\n") - fmt.Fprintf(w, "// max children %d (capacity %d)\n", maxChildren, 1<= 1<= 1<= 1< 0 && ss[0] == "" { - ss = ss[1:] - } - return ss -} - -// crush combines a list of strings, taking advantage of overlaps. It returns a -// single string that contains each input string as a substring. -func crush(ss []string) string { - maxLabelLen := 0 - for _, s := range ss { - if maxLabelLen < len(s) { - maxLabelLen = len(s) - } - } - - for prefixLen := maxLabelLen; prefixLen > 0; prefixLen-- { - prefixes := makePrefixMap(ss, prefixLen) - for i, s := range ss { - if len(s) <= prefixLen { - continue - } - mergeLabel(ss, i, prefixLen, prefixes) - } - } - - return strings.Join(ss, "") -} - -// mergeLabel merges the label at ss[i] with the first available matching label -// in prefixMap, where the last "prefixLen" characters in ss[i] match the first -// "prefixLen" characters in the matching label. -// It will merge ss[i] repeatedly until no more matches are available. -// All matching labels merged into ss[i] are replaced by "". -func mergeLabel(ss []string, i, prefixLen int, prefixes prefixMap) { - s := ss[i] - suffix := s[len(s)-prefixLen:] - for _, j := range prefixes[suffix] { - // Empty strings mean "already used." Also avoid merging with self. - if ss[j] == "" || i == j { - continue - } - if *v { - fmt.Fprintf(os.Stderr, "%d-length overlap at (%4d,%4d): %q and %q share %q\n", - prefixLen, i, j, ss[i], ss[j], suffix) - } - ss[i] += ss[j][prefixLen:] - ss[j] = "" - // ss[i] has a new suffix, so merge again if possible. - // Note: we only have to merge again at the same prefix length. Shorter - // prefix lengths will be handled in the next iteration of crush's for loop. - // Can there be matches for longer prefix lengths, introduced by the merge? - // I believe that any such matches would by necessity have been eliminated - // during substring removal or merged at a higher prefix length. For - // instance, in crush("abc", "cde", "bcdef"), combining "abc" and "cde" - // would yield "abcde", which could be merged with "bcdef." However, in - // practice "cde" would already have been elimintated by removeSubstrings. - mergeLabel(ss, i, prefixLen, prefixes) - return - } -} - -// prefixMap maps from a prefix to a list of strings containing that prefix. The -// list of strings is represented as indexes into a slice of strings stored -// elsewhere. -type prefixMap map[string][]int - -// makePrefixMap constructs a prefixMap from a slice of strings. -func makePrefixMap(ss []string, prefixLen int) prefixMap { - prefixes := make(prefixMap) - for i, s := range ss { - // We use < rather than <= because if a label matches on a prefix equal to - // its full length, that's actually a substring match handled by - // removeSubstrings. - if prefixLen < len(s) { - prefix := s[:prefixLen] - prefixes[prefix] = append(prefixes[prefix], i) - } - } - - return prefixes -} diff --git a/vendor/golang.org/x/net/publicsuffix/list.go b/vendor/golang.org/x/net/publicsuffix/list.go deleted file mode 100644 index 8bbf3bcd..00000000 --- a/vendor/golang.org/x/net/publicsuffix/list.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:generate go run gen.go - -// Package publicsuffix provides a public suffix list based on data from -// http://publicsuffix.org/. A public suffix is one under which Internet users -// can directly register names. -package publicsuffix // import "golang.org/x/net/publicsuffix" - -// TODO: specify case sensitivity and leading/trailing dot behavior for -// func PublicSuffix and func EffectiveTLDPlusOne. - -import ( - "fmt" - "net/http/cookiejar" - "strings" -) - -// List implements the cookiejar.PublicSuffixList interface by calling the -// PublicSuffix function. -var List cookiejar.PublicSuffixList = list{} - -type list struct{} - -func (list) PublicSuffix(domain string) string { - ps, _ := PublicSuffix(domain) - return ps -} - -func (list) String() string { - return version -} - -// PublicSuffix returns the public suffix of the domain using a copy of the -// publicsuffix.org database compiled into the library. -// -// icann is whether the public suffix is managed by the Internet Corporation -// for Assigned Names and Numbers. If not, the public suffix is privately -// managed. For example, foo.org and foo.co.uk are ICANN domains, -// foo.dyndns.org and foo.blogspot.co.uk are private domains. -// -// Use cases for distinguishing ICANN domains like foo.com from private -// domains like foo.appspot.com can be found at -// https://wiki.mozilla.org/Public_Suffix_List/Use_Cases -func PublicSuffix(domain string) (publicSuffix string, icann bool) { - lo, hi := uint32(0), uint32(numTLD) - s, suffix, wildcard := domain, len(domain), false -loop: - for { - dot := strings.LastIndex(s, ".") - if wildcard { - suffix = 1 + dot - } - if lo == hi { - break - } - f := find(s[1+dot:], lo, hi) - if f == notFound { - break - } - - u := nodes[f] >> (nodesBitsTextOffset + nodesBitsTextLength) - icann = u&(1<>= nodesBitsICANN - u = children[u&(1<>= childrenBitsLo - hi = u & (1<>= childrenBitsHi - switch u & (1<>= childrenBitsNodeType - wildcard = u&(1<>= nodesBitsTextLength - offset := x & (1<