vseledkin
9/13/2017 - 9:26 AM

ngram splitter

split space delimited string into ngrams

func SplitToNGrams(s string, n []int) []string {
	words := strings.Fields(s)
	var tokens []string
	for cursor, _ := range words {
		for _, ni := range n {
			// get suffix of length ni in words back from ni position in words array
			if cursor+ni <= len(words) {
				tokens = append(tokens, strings.Join(words[cursor:cursor+ni], " "))
			}
		}
	}
	return tokens
}