Switch from Godep to go vendoring

This commit is contained in:
Ken-Håvard Lieng 2016-03-01 01:51:26 +01:00
parent 6b37713bc0
commit cd317761c5
1504 changed files with 263076 additions and 34441 deletions

19
vendor/github.com/blevesearch/go-porterstemmer/LICENSE generated vendored Normal file
View file

@ -0,0 +1,19 @@
Copyright (c) 2013 Charles Iliya Krempeaux <charles@reptile.ca> :: http://changelog.ca/
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View file

@ -0,0 +1,118 @@
# This fork...
I'm maintaining this fork because the original author was not replying to issues or pull requests. For now I plan on maintaining this fork as necessary.
## Status
[![Build Status](https://travis-ci.org/blevesearch/go-porterstemmer.svg?branch=master)](https://travis-ci.org/blevesearch/go-porterstemmer)
[![Coverage Status](https://coveralls.io/repos/blevesearch/go-porterstemmer/badge.png?branch=HEAD)](https://coveralls.io/r/blevesearch/go-porterstemmer?branch=HEAD)
# Go Porter Stemmer
A native Go clean room implementation of the Porter Stemming Algorithm.
This algorithm is of interest to people doing Machine Learning or
Natural Language Processing (NLP).
This is NOT a port. This is a native Go implementation from the human-readable
description of the algorithm.
I've tried to make it (more) efficient by NOT internally using string's, but
instead internally using []rune's and using the same (array) buffer used by
the []rune slice (and sub-slices) at all steps of the algorithm.
For Porter Stemmer algorithm, see:
http://tartarus.org/martin/PorterStemmer/def.txt (URL #1)
http://tartarus.org/martin/PorterStemmer/ (URL #2)
# Departures
Also, since when I initially implemented it, it failed the tests at...
http://tartarus.org/martin/PorterStemmer/voc.txt (URL #3)
http://tartarus.org/martin/PorterStemmer/output.txt (URL #4)
... after reading the human-readble text over and over again to try to figure out
what the error I made was (and doing all sorts of things to debug it) I came to the
conclusion that the some of these tests were wrong according to the human-readable
description of the algorithm.
This led me to wonder if maybe other people's code that was passing these tests had
rules that were not in the human-readable description. Which led me to look at the source
code here...
http://tartarus.org/martin/PorterStemmer/c.txt (URL #5)
... When I looked there I noticed that there are some items marked as a "DEPARTURE",
which differ from the original algorithm. (There are 2 of these.)
I implemented these departures, and the tests at URL #3 and URL #4 all passed.
## Usage
To use this Golang library, use with something like:
package main
import (
"fmt"
"github.com/reiver/go-porterstemmer"
)
func main() {
word := "Waxes"
stem := porterstemmer.StemString(word)
fmt.Printf("The word [%s] has the stem [%s].\n", word, stem)
}
Alternatively, if you want to be a bit more efficient, use []rune slices instead, with code like:
package main
import (
"fmt"
"github.com/reiver/go-porterstemmer"
)
func main() {
word := []rune("Waxes")
stem := porterstemmer.Stem(word)
fmt.Printf("The word [%s] has the stem [%s].\n", string(word), string(stem))
}
Although NOTE that the above code may modify original slice (named "word" in the example) as a side
effect, for efficiency reasons. And that the slice named "stem" in the example above may be a
sub-slice of the slice named "word".
Also alternatively, if you already know that your word is already lowercase (and you don't need
this library to lowercase your word for you) you can instead use code like:
package main
import (
"fmt"
"github.com/reiver/go-porterstemmer"
)
func main() {
word := []rune("waxes")
stem := porterstemmer.StemWithoutLowerCasing(word)
fmt.Printf("The word [%s] has the stem [%s].\n", string(word), string(stem))
}
Again NOTE (like with the previous example) that the above code may modify original slice (named
"word" in the example) as a side effect, for efficiency reasons. And that the slice named "stem"
in the example above may be a sub-slice of the slice named "word".

View file

@ -0,0 +1,839 @@
package porterstemmer
import (
// "log"
"unicode"
)
func isConsonant(s []rune, i int) bool {
//DEBUG
//log.Printf("isConsonant: [%+v]", string(s[i]))
result := true
switch s[i] {
case 'a', 'e', 'i', 'o', 'u':
result = false
case 'y':
if 0 == i {
result = true
} else {
result = !isConsonant(s, i-1)
}
default:
result = true
}
return result
}
func measure(s []rune) uint {
// Initialize.
lenS := len(s)
result := uint(0)
i := 0
// Short Circuit.
if 0 == lenS {
/////////// RETURN
return result
}
// Ignore (potential) consonant sequence at the beginning of word.
for isConsonant(s, i) {
//DEBUG
//log.Printf("[measure([%s])] Eat Consonant [%d] -> [%s]", string(s), i, string(s[i]))
i++
if i >= lenS {
/////////////// RETURN
return result
}
}
// For each pair of a vowel sequence followed by a consonant sequence, increment result.
Outer:
for i < lenS {
for !isConsonant(s, i) {
//DEBUG
//log.Printf("[measure([%s])] VOWEL [%d] -> [%s]", string(s), i, string(s[i]))
i++
if i >= lenS {
/////////// BREAK
break Outer
}
}
for isConsonant(s, i) {
//DEBUG
//log.Printf("[measure([%s])] CONSONANT [%d] -> [%s]", string(s), i, string(s[i]))
i++
if i >= lenS {
result++
/////////// BREAK
break Outer
}
}
result++
}
// Return
return result
}
func hasSuffix(s, suffix []rune) bool {
lenSMinusOne := len(s) - 1
lenSuffixMinusOne := len(suffix) - 1
if lenSMinusOne <= lenSuffixMinusOne {
return false
} else if s[lenSMinusOne] != suffix[lenSuffixMinusOne] { // I suspect checking this first should speed this function up in practice.
/////// RETURN
return false
} else {
for i := 0; i < lenSuffixMinusOne; i++ {
if suffix[i] != s[lenSMinusOne-lenSuffixMinusOne+i] {
/////////////// RETURN
return false
}
}
}
return true
}
func containsVowel(s []rune) bool {
lenS := len(s)
for i := 0; i < lenS; i++ {
if !isConsonant(s, i) {
/////////// RETURN
return true
}
}
return false
}
func hasRepeatDoubleConsonantSuffix(s []rune) bool {
// Initialize.
lenS := len(s)
result := false
// Do it!
if 2 > lenS {
result = false
} else if s[lenS-1] == s[lenS-2] && isConsonant(s, lenS-1) { // Will using isConsonant() cause a problem with "YY"?
result = true
} else {
result = false
}
// Return,
return result
}
func hasConsonantVowelConsonantSuffix(s []rune) bool {
// Initialize.
lenS := len(s)
result := false
// Do it!
if 3 > lenS {
result = false
} else if isConsonant(s, lenS-3) && !isConsonant(s, lenS-2) && isConsonant(s, lenS-1) {
result = true
} else {
result = false
}
// Return
return result
}
func step1a(s []rune) []rune {
// Initialize.
var result []rune = s
lenS := len(s)
// Do it!
if suffix := []rune("sses"); hasSuffix(s, suffix) {
lenTrim := 2
subSlice := s[:lenS-lenTrim]
result = subSlice
} else if suffix := []rune("ies"); hasSuffix(s, suffix) {
lenTrim := 2
subSlice := s[:lenS-lenTrim]
result = subSlice
} else if suffix := []rune("ss"); hasSuffix(s, suffix) {
result = s
} else if suffix := []rune("s"); hasSuffix(s, suffix) {
lenSuffix := 1
subSlice := s[:lenS-lenSuffix]
result = subSlice
}
// Return.
return result
}
func step1b(s []rune) []rune {
// Initialize.
var result []rune = s
lenS := len(s)
// Do it!
if suffix := []rune("eed"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 0 < m {
lenTrim := 1
result = s[:lenS-lenTrim]
}
} else if suffix := []rune("ed"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
if containsVowel(subSlice) {
if suffix2 := []rune("at"); hasSuffix(subSlice, suffix2) {
lenTrim := -1
result = s[:lenS-lenSuffix-lenTrim]
} else if suffix2 := []rune("bl"); hasSuffix(subSlice, suffix2) {
lenTrim := -1
result = s[:lenS-lenSuffix-lenTrim]
} else if suffix2 := []rune("iz"); hasSuffix(subSlice, suffix2) {
lenTrim := -1
result = s[:lenS-lenSuffix-lenTrim]
} else if c := subSlice[len(subSlice)-1]; 'l' != c && 's' != c && 'z' != c && hasRepeatDoubleConsonantSuffix(subSlice) {
lenTrim := 1
lenSubSlice := len(subSlice)
result = subSlice[:lenSubSlice-lenTrim]
} else if c := subSlice[len(subSlice)-1]; 1 == measure(subSlice) && hasConsonantVowelConsonantSuffix(subSlice) && 'w' != c && 'x' != c && 'y' != c {
lenTrim := -1
result = s[:lenS-lenSuffix-lenTrim]
result[len(result)-1] = 'e'
} else {
result = subSlice
}
}
} else if suffix := []rune("ing"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
if containsVowel(subSlice) {
if suffix2 := []rune("at"); hasSuffix(subSlice, suffix2) {
lenTrim := -1
result = s[:lenS-lenSuffix-lenTrim]
result[len(result)-1] = 'e'
} else if suffix2 := []rune("bl"); hasSuffix(subSlice, suffix2) {
lenTrim := -1
result = s[:lenS-lenSuffix-lenTrim]
result[len(result)-1] = 'e'
} else if suffix2 := []rune("iz"); hasSuffix(subSlice, suffix2) {
lenTrim := -1
result = s[:lenS-lenSuffix-lenTrim]
result[len(result)-1] = 'e'
} else if c := subSlice[len(subSlice)-1]; 'l' != c && 's' != c && 'z' != c && hasRepeatDoubleConsonantSuffix(subSlice) {
lenTrim := 1
lenSubSlice := len(subSlice)
result = subSlice[:lenSubSlice-lenTrim]
} else if c := subSlice[len(subSlice)-1]; 1 == measure(subSlice) && hasConsonantVowelConsonantSuffix(subSlice) && 'w' != c && 'x' != c && 'y' != c {
lenTrim := -1
result = s[:lenS-lenSuffix-lenTrim]
result[len(result)-1] = 'e'
} else {
result = subSlice
}
}
}
// Return.
return result
}
func step1c(s []rune) []rune {
// Initialize.
lenS := len(s)
result := s
// Do it!
if 2 > lenS {
/////////// RETURN
return result
}
if 'y' == s[lenS-1] && containsVowel(s[:lenS-1]) {
result[lenS-1] = 'i'
} else if 'Y' == s[lenS-1] && containsVowel(s[:lenS-1]) {
result[lenS-1] = 'I'
}
// Return.
return result
}
func step2(s []rune) []rune {
// Initialize.
lenS := len(s)
result := s
// Do it!
if suffix := []rune("ational"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-5] = 'e'
result = result[:lenS-4]
}
} else if suffix := []rune("tional"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = result[:lenS-2]
}
} else if suffix := []rune("enci"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-1] = 'e'
}
} else if suffix := []rune("anci"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-1] = 'e'
}
} else if suffix := []rune("izer"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-1]
}
} else if suffix := []rune("bli"); hasSuffix(s, suffix) { // --DEPARTURE--
// } else if suffix := []rune("abli") ; hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-1] = 'e'
}
} else if suffix := []rune("alli"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-2]
}
} else if suffix := []rune("entli"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-2]
}
} else if suffix := []rune("eli"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-2]
}
} else if suffix := []rune("ousli"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-2]
}
} else if suffix := []rune("ization"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-5] = 'e'
result = s[:lenS-4]
}
} else if suffix := []rune("ation"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-3] = 'e'
result = s[:lenS-2]
}
} else if suffix := []rune("ator"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-2] = 'e'
result = s[:lenS-1]
}
} else if suffix := []rune("alism"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-3]
}
} else if suffix := []rune("iveness"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-4]
}
} else if suffix := []rune("fulness"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-4]
}
} else if suffix := []rune("ousness"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-4]
}
} else if suffix := []rune("aliti"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result = s[:lenS-3]
}
} else if suffix := []rune("iviti"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-3] = 'e'
result = result[:lenS-2]
}
} else if suffix := []rune("biliti"); hasSuffix(s, suffix) {
if 0 < measure(s[:lenS-len(suffix)]) {
result[lenS-5] = 'l'
result[lenS-4] = 'e'
result = result[:lenS-3]
}
} else if suffix := []rune("logi"); hasSuffix(s, suffix) { // --DEPARTURE--
if 0 < measure(s[:lenS-len(suffix)]) {
lenTrim := 1
result = s[:lenS-lenTrim]
}
}
// Return.
return result
}
func step3(s []rune) []rune {
// Initialize.
lenS := len(s)
result := s
// Do it!
if suffix := []rune("icate"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
if 0 < measure(s[:lenS-lenSuffix]) {
result = result[:lenS-3]
}
} else if suffix := []rune("ative"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 0 < m {
result = subSlice
}
} else if suffix := []rune("alize"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
if 0 < measure(s[:lenS-lenSuffix]) {
result = result[:lenS-3]
}
} else if suffix := []rune("iciti"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
if 0 < measure(s[:lenS-lenSuffix]) {
result = result[:lenS-3]
}
} else if suffix := []rune("ical"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
if 0 < measure(s[:lenS-lenSuffix]) {
result = result[:lenS-2]
}
} else if suffix := []rune("ful"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 0 < m {
result = subSlice
}
} else if suffix := []rune("ness"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 0 < m {
result = subSlice
}
}
// Return.
return result
}
func step4(s []rune) []rune {
// Initialize.
lenS := len(s)
result := s
// Do it!
if suffix := []rune("al"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = result[:lenS-lenSuffix]
}
} else if suffix := []rune("ance"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = result[:lenS-lenSuffix]
}
} else if suffix := []rune("ence"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = result[:lenS-lenSuffix]
}
} else if suffix := []rune("er"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ic"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("able"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ible"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ant"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ement"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ment"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ent"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ion"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
c := subSlice[len(subSlice)-1]
if 1 < m && ('s' == c || 't' == c) {
result = subSlice
}
} else if suffix := []rune("ou"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ism"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ate"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("iti"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ous"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ive"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
} else if suffix := []rune("ize"); hasSuffix(s, suffix) {
lenSuffix := len(suffix)
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
}
// Return.
return result
}
func step5a(s []rune) []rune {
// Initialize.
lenS := len(s)
result := s
// Do it!
if 'e' == s[lenS-1] {
lenSuffix := 1
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
} else if 1 == m {
if c := subSlice[len(subSlice)-1]; !(hasConsonantVowelConsonantSuffix(subSlice) && 'w' != c && 'x' != c && 'y' != c) {
result = subSlice
}
}
}
// Return.
return result
}
func step5b(s []rune) []rune {
// Initialize.
lenS := len(s)
result := s
// Do it!
if 2 < lenS && 'l' == s[lenS-2] && 'l' == s[lenS-1] {
lenSuffix := 1
subSlice := s[:lenS-lenSuffix]
m := measure(subSlice)
if 1 < m {
result = subSlice
}
}
// Return.
return result
}
func StemString(s string) string {
// Convert string to []rune
runeArr := []rune(s)
// Stem.
runeArr = Stem(runeArr)
// Convert []rune to string
str := string(runeArr)
// Return.
return str
}
func Stem(s []rune) []rune {
// Initialize.
lenS := len(s)
// Short circuit.
if 0 == lenS {
/////////// RETURN
return s
}
// Make all runes lowercase.
for i := 0; i < lenS; i++ {
s[i] = unicode.ToLower(s[i])
}
// Stem
result := StemWithoutLowerCasing(s)
// Return.
return result
}
func StemWithoutLowerCasing(s []rune) []rune {
// Initialize.
lenS := len(s)
// Words that are of length 2 or less is already stemmed.
// Don't do anything.
if 2 >= lenS {
/////////// RETURN
return s
}
// Stem
s = step1a(s)
s = step1b(s)
s = step1c(s)
s = step2(s)
s = step3(s)
s = step4(s)
s = step5a(s)
s = step5b(s)
// Return.
return s
}

View file

@ -0,0 +1,57 @@
package porterstemmer
import (
"testing"
)
func TestContainsVowel(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected bool
}, 15)
tests[i].S = []rune("apple")
tests[i].Expected = true
i++
tests[i].S = []rune("f")
tests[i].Expected = false
i++
tests[i].S = []rune("a")
tests[i].Expected = true
i++
tests[i].S = []rune("e")
tests[i].Expected = true
i++
tests[i].S = []rune("i")
tests[i].Expected = true
i++
tests[i].S = []rune("o")
tests[i].Expected = true
i++
tests[i].S = []rune("u")
tests[i].Expected = true
i++
tests[i].S = []rune("y")
tests[i].Expected = false
i++
tests[i].S = []rune("cy")
tests[i].Expected = true
i++
for _, datum := range tests {
if actual := containsVowel(datum.S); actual != datum.Expected {
t.Errorf("Did NOT get what was expected for calling containsVowel() on [%s]. Expect [%t] but got [%t]", string(datum.S), datum.Expected, actual)
}
}
}

View file

@ -0,0 +1,20 @@
package porterstemmer
import (
"testing"
)
// Test for issue listed here:
// https://github.com/reiver/go-porterstemmer/issues/1
//
// StemString("ion") was causing runtime exception
func TestStemStringIon(t *testing.T) {
expected := "ion"
s := "ion"
actual := StemString(s)
if expected != actual {
t.Errorf("Input: [%s] -> Actual: [%s]. Expected: [%s]", s, actual, expected)
}
}

View file

@ -0,0 +1,59 @@
package porterstemmer
import (
"bytes"
"testing"
)
const maxFuzzLen = 6
// Test inputs of English characters less than maxFuzzLen
// Added to help diagnose https://github.com/reiver/go-porterstemmer/issues/4
func TestStemFuzz(t *testing.T) {
input := []byte{'a'}
for len(input) < maxFuzzLen {
// test input
panicked := false
func() {
defer func() { panicked = recover() != nil }()
StemString(string(input))
}()
if panicked {
t.Errorf("StemString panicked for input '%s'", input)
}
// if all z's extend
if allZs(input) {
input = bytes.Repeat([]byte{'a'}, len(input)+1)
} else {
// increment
input = incrementBytes(input)
}
}
}
func incrementBytes(in []byte) []byte {
rv := make([]byte, len(in))
copy(rv, in)
for i := len(rv) - 1; i >= 0; i-- {
if rv[i]+1 == '{' {
rv[i] = 'a'
continue
}
rv[i] = rv[i] + 1
break
}
return rv
}
func allZs(in []byte) bool {
for _, b := range in {
if b != 'z' {
return false
}
}
return true
}

View file

@ -0,0 +1,42 @@
package porterstemmer
import (
"testing"
)
func TestHasDoubleConsonantSuffix(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected bool
}, 12)
tests[i].S = []rune("apple")
tests[i].Expected = false
i++
tests[i].S = []rune("hiss")
tests[i].Expected = true
i++
tests[i].S = []rune("fizz")
tests[i].Expected = true
i++
tests[i].S = []rune("fill")
tests[i].Expected = true
i++
tests[i].S = []rune("ahaa")
tests[i].Expected = false
i++
for _, datum := range tests {
if actual := hasRepeatDoubleConsonantSuffix(datum.S); actual != datum.Expected {
t.Errorf("Did NOT get what was expected for calling hasDoubleConsonantSuffix() on [%s]. Expect [%t] but got [%t]", string(datum.S), datum.Expected, actual)
}
}
}

View file

@ -0,0 +1,432 @@
package porterstemmer
import (
"testing"
)
func TestHasSuffix(t *testing.T) {
tests := make([]struct {
S []rune
Suffix []rune
Expected bool
}, 82)
i := 0
tests[i].S = []rune("ran")
tests[i].Suffix = []rune("er")
tests[i].Expected = false
i++
tests[i].S = []rune("runner")
tests[i].Suffix = []rune("er")
tests[i].Expected = true
i++
tests[i].S = []rune("runnar")
tests[i].Suffix = []rune("er")
tests[i].Expected = false
i++
tests[i].S = []rune("runned")
tests[i].Suffix = []rune("er")
tests[i].Expected = false
i++
tests[i].S = []rune("runnre")
tests[i].Suffix = []rune("er")
tests[i].Expected = false
i++
// FIXME marty changed Expected
// to false here because it seems
// the contract does not support
// suffix of same length as input
// as this test implied
tests[i].S = []rune("er")
tests[i].Suffix = []rune("er")
tests[i].Expected = false
i++
tests[i].S = []rune("re")
tests[i].Suffix = []rune("er")
tests[i].Expected = false
i++
tests[i].S = []rune("ran")
tests[i].Suffix = []rune("ER")
tests[i].Expected = false
i++
tests[i].S = []rune("runner")
tests[i].Suffix = []rune("ER")
tests[i].Expected = false
i++
tests[i].S = []rune("runnar")
tests[i].Suffix = []rune("ER")
tests[i].Expected = false
i++
tests[i].S = []rune("runned")
tests[i].Suffix = []rune("ER")
tests[i].Expected = false
i++
tests[i].S = []rune("runnre")
tests[i].Suffix = []rune("ER")
tests[i].Expected = false
i++
tests[i].S = []rune("er")
tests[i].Suffix = []rune("ER")
tests[i].Expected = false
i++
tests[i].S = []rune("re")
tests[i].Suffix = []rune("ER")
tests[i].Expected = false
i++
tests[i].S = []rune("")
tests[i].Suffix = []rune("er")
tests[i].Expected = false
i++
tests[i].S = []rune("e")
tests[i].Suffix = []rune("er")
tests[i].Expected = false
i++
tests[i].S = []rune("caresses")
tests[i].Suffix = []rune("sses")
tests[i].Expected = true
i++
tests[i].S = []rune("ponies")
tests[i].Suffix = []rune("ies")
tests[i].Expected = true
i++
tests[i].S = []rune("caress")
tests[i].Suffix = []rune("ss")
tests[i].Expected = true
i++
tests[i].S = []rune("cats")
tests[i].Suffix = []rune("s")
tests[i].Expected = true
i++
tests[i].S = []rune("feed")
tests[i].Suffix = []rune("eed")
tests[i].Expected = true
i++
tests[i].S = []rune("agreed")
tests[i].Suffix = []rune("eed")
tests[i].Expected = true
i++
tests[i].S = []rune("plastered")
tests[i].Suffix = []rune("ed")
tests[i].Expected = true
i++
tests[i].S = []rune("bled")
tests[i].Suffix = []rune("ed")
tests[i].Expected = true
i++
tests[i].S = []rune("motoring")
tests[i].Suffix = []rune("ing")
tests[i].Expected = true
i++
tests[i].S = []rune("sing")
tests[i].Suffix = []rune("ing")
tests[i].Expected = true
i++
tests[i].S = []rune("conflat")
tests[i].Suffix = []rune("at")
tests[i].Expected = true
i++
tests[i].S = []rune("troubl")
tests[i].Suffix = []rune("bl")
tests[i].Expected = true
i++
tests[i].S = []rune("siz")
tests[i].Suffix = []rune("iz")
tests[i].Expected = true
i++
tests[i].S = []rune("happy")
tests[i].Suffix = []rune("y")
tests[i].Expected = true
i++
tests[i].S = []rune("sky")
tests[i].Suffix = []rune("y")
tests[i].Expected = true
i++
tests[i].S = []rune("relational")
tests[i].Suffix = []rune("ational")
tests[i].Expected = true
i++
tests[i].S = []rune("conditional")
tests[i].Suffix = []rune("tional")
tests[i].Expected = true
i++
tests[i].S = []rune("rational")
tests[i].Suffix = []rune("tional")
tests[i].Expected = true
i++
tests[i].S = []rune("valenci")
tests[i].Suffix = []rune("enci")
tests[i].Expected = true
i++
tests[i].S = []rune("hesitanci")
tests[i].Suffix = []rune("anci")
tests[i].Expected = true
i++
tests[i].S = []rune("digitizer")
tests[i].Suffix = []rune("izer")
tests[i].Expected = true
i++
tests[i].S = []rune("conformabli")
tests[i].Suffix = []rune("abli")
tests[i].Expected = true
i++
tests[i].S = []rune("radicalli")
tests[i].Suffix = []rune("alli")
tests[i].Expected = true
i++
tests[i].S = []rune("differentli")
tests[i].Suffix = []rune("entli")
tests[i].Expected = true
i++
tests[i].S = []rune("vileli")
tests[i].Suffix = []rune("eli")
tests[i].Expected = true
i++
tests[i].S = []rune("analogousli")
tests[i].Suffix = []rune("ousli")
tests[i].Expected = true
i++
tests[i].S = []rune("vietnamization")
tests[i].Suffix = []rune("ization")
tests[i].Expected = true
i++
tests[i].S = []rune("predication")
tests[i].Suffix = []rune("ation")
tests[i].Expected = true
i++
tests[i].S = []rune("operator")
tests[i].Suffix = []rune("ator")
tests[i].Expected = true
i++
tests[i].S = []rune("feudalism")
tests[i].Suffix = []rune("alism")
tests[i].Expected = true
i++
tests[i].S = []rune("decisiveness")
tests[i].Suffix = []rune("iveness")
tests[i].Expected = true
i++
tests[i].S = []rune("hopefulness")
tests[i].Suffix = []rune("fulness")
tests[i].Expected = true
i++
tests[i].S = []rune("callousness")
tests[i].Suffix = []rune("ousness")
tests[i].Expected = true
i++
tests[i].S = []rune("formaliti")
tests[i].Suffix = []rune("aliti")
tests[i].Expected = true
i++
tests[i].S = []rune("sensitiviti")
tests[i].Suffix = []rune("iviti")
tests[i].Expected = true
i++
tests[i].S = []rune("sensibiliti")
tests[i].Suffix = []rune("biliti")
tests[i].Expected = true
i++
tests[i].S = []rune("triplicate")
tests[i].Suffix = []rune("icate")
tests[i].Expected = true
i++
tests[i].S = []rune("formative")
tests[i].Suffix = []rune("ative")
tests[i].Expected = true
i++
tests[i].S = []rune("formalize")
tests[i].Suffix = []rune("alize")
tests[i].Expected = true
i++
tests[i].S = []rune("electriciti")
tests[i].Suffix = []rune("iciti")
tests[i].Expected = true
i++
tests[i].S = []rune("electrical")
tests[i].Suffix = []rune("ical")
tests[i].Expected = true
i++
tests[i].S = []rune("hopeful")
tests[i].Suffix = []rune("ful")
tests[i].Expected = true
i++
tests[i].S = []rune("goodness")
tests[i].Suffix = []rune("ness")
tests[i].Expected = true
i++
tests[i].S = []rune("revival")
tests[i].Suffix = []rune("al")
tests[i].Expected = true
i++
tests[i].S = []rune("allowance")
tests[i].Suffix = []rune("ance")
tests[i].Expected = true
i++
tests[i].S = []rune("inference")
tests[i].Suffix = []rune("ence")
tests[i].Expected = true
i++
tests[i].S = []rune("airliner")
tests[i].Suffix = []rune("er")
tests[i].Expected = true
i++
tests[i].S = []rune("gyroscopic")
tests[i].Suffix = []rune("ic")
tests[i].Expected = true
i++
tests[i].S = []rune("adjustable")
tests[i].Suffix = []rune("able")
tests[i].Expected = true
i++
tests[i].S = []rune("defensible")
tests[i].Suffix = []rune("ible")
tests[i].Expected = true
i++
tests[i].S = []rune("irritant")
tests[i].Suffix = []rune("ant")
tests[i].Expected = true
i++
tests[i].S = []rune("replacement")
tests[i].Suffix = []rune("ement")
tests[i].Expected = true
i++
tests[i].S = []rune("adjustment")
tests[i].Suffix = []rune("ment")
tests[i].Expected = true
i++
tests[i].S = []rune("dependent")
tests[i].Suffix = []rune("ent")
tests[i].Expected = true
i++
tests[i].S = []rune("adoption")
tests[i].Suffix = []rune("ion")
tests[i].Expected = true
i++
tests[i].S = []rune("homologou")
tests[i].Suffix = []rune("ou")
tests[i].Expected = true
i++
tests[i].S = []rune("communism")
tests[i].Suffix = []rune("ism")
tests[i].Expected = true
i++
tests[i].S = []rune("activate")
tests[i].Suffix = []rune("ate")
tests[i].Expected = true
i++
tests[i].S = []rune("angulariti")
tests[i].Suffix = []rune("iti")
tests[i].Expected = true
i++
tests[i].S = []rune("homologous")
tests[i].Suffix = []rune("ous")
tests[i].Expected = true
i++
tests[i].S = []rune("effective")
tests[i].Suffix = []rune("ive")
tests[i].Expected = true
i++
tests[i].S = []rune("bowdlerize")
tests[i].Suffix = []rune("ize")
tests[i].Expected = true
i++
tests[i].S = []rune("probate")
tests[i].Suffix = []rune("e")
tests[i].Expected = true
i++
tests[i].S = []rune("rate")
tests[i].Suffix = []rune("e")
tests[i].Expected = true
i++
tests[i].S = []rune("cease")
tests[i].Suffix = []rune("e")
tests[i].Expected = true
i++
for _, datum := range tests {
if actual := hasSuffix(datum.S, datum.Suffix); actual != datum.Expected {
t.Errorf("Did NOT get what was expected for calling hasSuffix() on [%s] with suffix [%s]. Expect [%t] but got [%t]", string(datum.S), string(datum.Suffix), datum.Expected, actual)
}
}
}

View file

@ -0,0 +1,74 @@
package porterstemmer
import (
"testing"
)
func TestIsConsontant(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []bool
}, 12)
tests[i].S = []rune("apple")
tests[i].Expected = []bool{false, true, true, true, false}
i++
tests[i].S = []rune("cyan")
tests[i].Expected = []bool{true, false, false, true}
i++
tests[i].S = []rune("connects")
tests[i].Expected = []bool{true, false, true, true, false, true, true, true}
i++
tests[i].S = []rune("yellow")
tests[i].Expected = []bool{true, false, true, true, false, true}
i++
tests[i].S = []rune("excellent")
tests[i].Expected = []bool{false, true, true, false, true, true, false, true, true}
i++
tests[i].S = []rune("yuk")
tests[i].Expected = []bool{true, false, true}
i++
tests[i].S = []rune("syzygy")
tests[i].Expected = []bool{true, false, true, false, true, false}
i++
tests[i].S = []rune("school")
tests[i].Expected = []bool{true, true, true, false, false, true}
i++
tests[i].S = []rune("pay")
tests[i].Expected = []bool{true, false, true}
i++
tests[i].S = []rune("golang")
tests[i].Expected = []bool{true, false, true, false, true, true}
i++
// NOTE: The Porter Stemmer technical should make a mistake on the second "y".
// Really, both the 1st and 2nd "y" are consontants. But
tests[i].S = []rune("sayyid")
tests[i].Expected = []bool{true, false, true, false, false, true}
i++
tests[i].S = []rune("ya")
tests[i].Expected = []bool{true, false}
i++
for _, datum := range tests {
for i = 0; i < len(datum.S); i++ {
if actual := isConsonant(datum.S, i); actual != datum.Expected[i] {
t.Errorf("Did NOT get what was expected for calling isConsonant() on [%s] at [%d] (i.e., [%s]). Expect [%t] but got [%t]", string(datum.S), i, string(datum.S[i]), datum.Expected[i], actual)
}
} // for
}
}

View file

@ -0,0 +1,99 @@
package porterstemmer
import (
"testing"
)
func TestMeasure(t *testing.T) {
tests := make([]struct {
S []rune
Expected uint
}, 27)
tests[0].S = []rune("ya")
tests[0].Expected = 0
tests[1].S = []rune("cyan")
tests[1].Expected = 1
tests[2].S = []rune("connects")
tests[2].Expected = 2
tests[3].S = []rune("yellow")
tests[3].Expected = 2
tests[4].S = []rune("excellent")
tests[4].Expected = 3
tests[5].S = []rune("yuk")
tests[5].Expected = 1
tests[6].S = []rune("syzygy")
tests[6].Expected = 2
tests[7].S = []rune("school")
tests[7].Expected = 1
tests[8].S = []rune("pay")
tests[8].Expected = 1
tests[9].S = []rune("golang")
tests[9].Expected = 2
// NOTE: The Porter Stemmer technical should make a mistake on the second "y".
// Really, both the 1st and 2nd "y" are consontants. But
tests[10].S = []rune("sayyid")
tests[10].Expected = 2
tests[11].S = []rune("ya")
tests[11].Expected = 0
tests[12].S = []rune("")
tests[12].Expected = 0
tests[13].S = []rune("tr")
tests[13].Expected = 0
tests[14].S = []rune("ee")
tests[14].Expected = 0
tests[15].S = []rune("tree")
tests[15].Expected = 0
tests[16].S = []rune("t")
tests[16].Expected = 0
tests[18].S = []rune("by")
tests[18].Expected = 0
tests[19].S = []rune("trouble")
tests[19].Expected = 1
tests[20].S = []rune("oats")
tests[20].Expected = 1
tests[21].S = []rune("trees")
tests[21].Expected = 1
tests[22].S = []rune("ivy")
tests[22].Expected = 1
tests[23].S = []rune("troubles")
tests[23].Expected = 2
tests[24].S = []rune("private")
tests[24].Expected = 2
tests[25].S = []rune("oaten")
tests[25].Expected = 2
tests[26].S = []rune("orrery")
tests[26].Expected = 2
for _, datum := range tests {
if actual := measure(datum.S); actual != datum.Expected {
t.Errorf("Did NOT get what was expected for calling measure() on [%s]. Expect [%d] but got [%d]", string(datum.S), datum.Expected, actual)
}
}
}

View file

@ -0,0 +1,118 @@
package porterstemmer
import (
"bufio"
"io/ioutil"
"net/http"
"os"
"strings"
"testing"
)
func TestStemString(t *testing.T) {
testDataDirName := "testdata"
_, err := os.Stat(testDataDirName)
if nil != err {
_ = os.Mkdir(testDataDirName, 0755)
}
_, err = os.Stat(testDataDirName)
if nil != err {
t.Errorf("The test data folder ([%s]) does not exists (and could not create it). Received error: [%v]", testDataDirName, err)
/////// RETURN
return
}
vocFileName := testDataDirName + "/voc.txt"
_, err = os.Stat(vocFileName)
if nil != err {
vocHref := "http://tartarus.org/martin/PorterStemmer/voc.txt"
resp, err := http.Get(vocHref)
if nil != err {
t.Errorf("Could not download test file (from web) from URL: [%s]. Received error: [%v]", vocHref, err)
/////////// RETURN
return
}
respBody, err := ioutil.ReadAll(resp.Body)
if nil != err {
t.Errorf("Error loading the contents of from URL: [%s]. Received error: [%v].", vocHref, err)
/////////// RETURN
return
}
_ = ioutil.WriteFile(vocFileName, respBody, 0644)
}
vocFd, err := os.Open(vocFileName)
if nil != err {
t.Errorf("Could NOT open testdata file: [%s]. Received error: [%v]", vocFileName, err)
/////// RETURN
return
}
defer vocFd.Close()
voc := bufio.NewReaderSize(vocFd, 1024)
outFileName := testDataDirName + "/output.txt"
_, err = os.Stat(outFileName)
if nil != err {
outHref := "http://tartarus.org/martin/PorterStemmer/output.txt"
resp, err := http.Get(outHref)
if nil != err {
t.Errorf("Could not download test file (from web) from URL: [%s]. Received error: [%v]", outHref, err)
/////////// RETURN
return
}
respBody, err := ioutil.ReadAll(resp.Body)
if nil != err {
t.Errorf("Error loading the contents of from URL: [%s]. Received error: [%v].", outHref, err)
/////////// RETURN
return
}
_ = ioutil.WriteFile(outFileName, respBody, 0644)
}
outFd, err := os.Open(outFileName)
if nil != err {
t.Errorf("Could NOT open testdata file: [%s]. Received error: [%v]", outFileName, err)
/////// RETURN
return
}
defer outFd.Close()
out := bufio.NewReaderSize(outFd, 1024)
for {
vocS, err := voc.ReadString('\n')
if nil != err {
/////// BREAK
break
}
vocS = strings.Trim(vocS, "\n\r\t ")
expected, err := out.ReadString('\n')
if nil != err {
t.Errorf("Received unexpected error when trying to read a line from [%s]. Received error: [%v]", outFileName, err)
/////// BREAK
break
}
expected = strings.Trim(expected, "\n\r\t ")
actual := StemString(vocS)
if expected != actual {
t.Errorf("Input: [%s] -> Actual: [%s]. Expected: [%s]", vocS, actual, expected)
}
}
}

View file

@ -0,0 +1,56 @@
package porterstemmer
import (
"testing"
)
func TestStemWithoutLowerCasing(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 3)
tests[i].S = []rune("controll")
tests[i].Expected = []rune("control")
i++
tests[i].S = []rune("roll")
tests[i].Expected = []rune("roll")
i++
for _, datum := range tests {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = StemWithoutLowerCasing(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling StemWithoutLowerCasing() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
}
}

View file

@ -0,0 +1,70 @@
package porterstemmer
import (
"testing"
)
func TestStep1a(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 12)
tests[i].S = []rune("caresses")
tests[i].Expected = []rune("caress")
i++
tests[i].S = []rune("ponies")
tests[i].Expected = []rune("poni")
i++
tests[i].S = []rune("ties")
tests[i].Expected = []rune("ti")
i++
tests[i].S = []rune("caress")
tests[i].Expected = []rune("caress")
i++
tests[i].S = []rune("cats")
tests[i].Expected = []rune("cat")
i++
for _, datum := range tests {
for i = 0; i < len(datum.S); i++ {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = step1a(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling step1a() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
} // for
}
}

View file

@ -0,0 +1,112 @@
package porterstemmer
import (
"testing"
)
func TestStep1b(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 17)
tests[i].S = []rune("feed")
tests[i].Expected = []rune("feed")
i++
tests[i].S = []rune("agreed")
tests[i].Expected = []rune("agree")
i++
tests[i].S = []rune("plastered")
tests[i].Expected = []rune("plaster")
i++
tests[i].S = []rune("bled")
tests[i].Expected = []rune("bled")
i++
tests[i].S = []rune("motoring")
tests[i].Expected = []rune("motor")
i++
tests[i].S = []rune("sing")
tests[i].Expected = []rune("sing")
i++
tests[i].S = []rune("conflated")
tests[i].Expected = []rune("conflate")
i++
tests[i].S = []rune("troubled")
tests[i].Expected = []rune("trouble")
i++
tests[i].S = []rune("sized")
tests[i].Expected = []rune("size")
i++
tests[i].S = []rune("hopping")
tests[i].Expected = []rune("hop")
i++
tests[i].S = []rune("tanned")
tests[i].Expected = []rune("tan")
i++
tests[i].S = []rune("falling")
tests[i].Expected = []rune("fall")
i++
tests[i].S = []rune("hissing")
tests[i].Expected = []rune("hiss")
i++
tests[i].S = []rune("fizzed")
tests[i].Expected = []rune("fizz")
i++
tests[i].S = []rune("failing")
tests[i].Expected = []rune("fail")
i++
tests[i].S = []rune("filing")
tests[i].Expected = []rune("file")
i++
for _, datum := range tests {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = step1b(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling step1b() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
}
}

View file

@ -0,0 +1,60 @@
package porterstemmer
import (
"testing"
)
func TestStep1c(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 17)
tests[i].S = []rune("happy")
tests[i].Expected = []rune("happi")
i++
tests[i].S = []rune("sky")
tests[i].Expected = []rune("sky")
i++
tests[i].S = []rune("apology")
tests[i].Expected = []rune("apologi")
i++
for _, datum := range tests {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = step1c(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling step1c() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
}
}

View file

@ -0,0 +1,132 @@
package porterstemmer
import (
"testing"
)
func TestStep2(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 22)
tests[i].S = []rune("relational")
tests[i].Expected = []rune("relate")
i++
tests[i].S = []rune("conditional")
tests[i].Expected = []rune("condition")
i++
tests[i].S = []rune("rational")
tests[i].Expected = []rune("rational")
i++
tests[i].S = []rune("valenci")
tests[i].Expected = []rune("valence")
i++
tests[i].S = []rune("hesitanci")
tests[i].Expected = []rune("hesitance")
i++
tests[i].S = []rune("digitizer")
tests[i].Expected = []rune("digitize")
i++
tests[i].S = []rune("conformabli")
tests[i].Expected = []rune("conformable")
i++
tests[i].S = []rune("radicalli")
tests[i].Expected = []rune("radical")
i++
tests[i].S = []rune("differentli")
tests[i].Expected = []rune("different")
i++
tests[i].S = []rune("vileli")
tests[i].Expected = []rune("vile")
i++
tests[i].S = []rune("analogousli")
tests[i].Expected = []rune("analogous")
i++
tests[i].S = []rune("vietnamization")
tests[i].Expected = []rune("vietnamize")
i++
tests[i].S = []rune("predication")
tests[i].Expected = []rune("predicate")
i++
tests[i].S = []rune("operator")
tests[i].Expected = []rune("operate")
i++
tests[i].S = []rune("feudalism")
tests[i].Expected = []rune("feudal")
i++
tests[i].S = []rune("decisiveness")
tests[i].Expected = []rune("decisive")
i++
tests[i].S = []rune("hopefulness")
tests[i].Expected = []rune("hopeful")
i++
tests[i].S = []rune("callousness")
tests[i].Expected = []rune("callous")
i++
tests[i].S = []rune("formaliti")
tests[i].Expected = []rune("formal")
i++
tests[i].S = []rune("sensitiviti")
tests[i].Expected = []rune("sensitive")
i++
tests[i].S = []rune("sensibiliti")
tests[i].Expected = []rune("sensible")
i++
for _, datum := range tests {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = step2(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling step2() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
}
}

View file

@ -0,0 +1,76 @@
package porterstemmer
import (
"testing"
)
func TestStep3(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 22)
tests[i].S = []rune("triplicate")
tests[i].Expected = []rune("triplic")
i++
tests[i].S = []rune("formative")
tests[i].Expected = []rune("form")
i++
tests[i].S = []rune("formalize")
tests[i].Expected = []rune("formal")
i++
tests[i].S = []rune("electriciti")
tests[i].Expected = []rune("electric")
i++
tests[i].S = []rune("electrical")
tests[i].Expected = []rune("electric")
i++
tests[i].S = []rune("hopeful")
tests[i].Expected = []rune("hope")
i++
tests[i].S = []rune("goodness")
tests[i].Expected = []rune("good")
i++
for _, datum := range tests {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = step3(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling step3() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
}
}

View file

@ -0,0 +1,124 @@
package porterstemmer
import (
"testing"
)
func TestStep4(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 20)
tests[i].S = []rune("revival")
tests[i].Expected = []rune("reviv")
i++
tests[i].S = []rune("allowance")
tests[i].Expected = []rune("allow")
i++
tests[i].S = []rune("inference")
tests[i].Expected = []rune("infer")
i++
tests[i].S = []rune("airliner")
tests[i].Expected = []rune("airlin")
i++
tests[i].S = []rune("gyroscopic")
tests[i].Expected = []rune("gyroscop")
i++
tests[i].S = []rune("adjustable")
tests[i].Expected = []rune("adjust")
i++
tests[i].S = []rune("defensible")
tests[i].Expected = []rune("defens")
i++
tests[i].S = []rune("irritant")
tests[i].Expected = []rune("irrit")
i++
tests[i].S = []rune("replacement")
tests[i].Expected = []rune("replac")
i++
tests[i].S = []rune("adjustment")
tests[i].Expected = []rune("adjust")
i++
tests[i].S = []rune("dependent")
tests[i].Expected = []rune("depend")
i++
tests[i].S = []rune("adoption")
tests[i].Expected = []rune("adopt")
i++
tests[i].S = []rune("homologou")
tests[i].Expected = []rune("homolog")
i++
tests[i].S = []rune("communism")
tests[i].Expected = []rune("commun")
i++
tests[i].S = []rune("activate")
tests[i].Expected = []rune("activ")
i++
tests[i].S = []rune("angulariti")
tests[i].Expected = []rune("angular")
i++
tests[i].S = []rune("homologous")
tests[i].Expected = []rune("homolog")
i++
tests[i].S = []rune("effective")
tests[i].Expected = []rune("effect")
i++
tests[i].S = []rune("bowdlerize")
tests[i].Expected = []rune("bowdler")
i++
for _, datum := range tests {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = step4(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling step4() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
}
}

View file

@ -0,0 +1,60 @@
package porterstemmer
import (
"testing"
)
func TestStep5a(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 3)
tests[i].S = []rune("probate")
tests[i].Expected = []rune("probat")
i++
tests[i].S = []rune("rate")
tests[i].Expected = []rune("rate")
i++
tests[i].S = []rune("cease")
tests[i].Expected = []rune("ceas")
i++
for _, datum := range tests {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = step5a(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling step5a() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
}
}

View file

@ -0,0 +1,56 @@
package porterstemmer
import (
"testing"
)
func TestStep5b(t *testing.T) {
i := 0
tests := make([]struct {
S []rune
Expected []rune
}, 3)
tests[i].S = []rune("controll")
tests[i].Expected = []rune("control")
i++
tests[i].S = []rune("roll")
tests[i].Expected = []rune("roll")
i++
for _, datum := range tests {
actual := make([]rune, len(datum.S))
copy(actual, datum.S)
actual = step5b(actual)
lenActual := len(actual)
lenExpected := len(datum.Expected)
equal := true
if 0 == lenActual && 0 == lenExpected {
equal = true
} else if lenActual != lenExpected {
equal = false
} else if actual[0] != datum.Expected[0] {
equal = false
} else if actual[lenActual-1] != datum.Expected[lenExpected-1] {
equal = false
} else {
for j := 0; j < lenActual; j++ {
if actual[j] != datum.Expected[j] {
equal = false
}
}
}
if !equal {
t.Errorf("Did NOT get what was expected for calling step5b() on [%s]. Expect [%s] but got [%s]", string(datum.S), string(datum.Expected), string(actual))
}
}
}