test_cpp_string_tokenize-00.cpp
#include <iostream>
#include <iterator>
#include <set>
#include <sstream>
#include <string>
namespace {
const std::string text("lonely lovely baboon sat on his ass an he was a very lovely but lonely too most likely the loveliest baboon though his ass wasn't lovely really");
template<typename T>
std::ostream& operator<<(std::ostream& out, const std::set<T>& s) {
std::copy(s.begin(),
s.end(),
std::ostream_iterator<T>(out, " "));
return out;
}
}
int main(int argc, char* argv[]) {
std::cout << "<input>" << text << "</input>" << std::endl;
std::istringstream in(text);
std::set<std::string> uniq;
for(std::string s;
std::getline(in, s, ' ');
uniq.insert(s));
std::cout << "<output>" << uniq << "</output>" << std::endl;
}