00001 #include "util/tokenize.hh"
00002
00003 #define BOOST_TEST_MODULE TokenizeTest
00004 #include <boost/test/unit_test.hpp>
00005
00006 namespace util
00007 {
00008 namespace
00009 {
00010
00011 BOOST_AUTO_TEST_CASE(empty_text_yields_empty_vector)
00012 {
00013 const std::vector<std::string> tokens = util::tokenize("");
00014 BOOST_CHECK_EQUAL(tokens.size(), 0);
00015 }
00016
00017 BOOST_AUTO_TEST_CASE(whitespace_only_yields_empty_vector)
00018 {
00019 const std::vector<std::string> tokens = util::tokenize(" ");
00020 BOOST_CHECK_EQUAL(tokens.size(), 0);
00021 }
00022
00023 BOOST_AUTO_TEST_CASE(parses_single_token)
00024 {
00025 const std::vector<std::string> tokens = util::tokenize("mytoken");
00026 BOOST_CHECK_EQUAL(tokens.size(), 1);
00027 BOOST_CHECK_EQUAL(tokens[0], "mytoken");
00028 }
00029
00030 BOOST_AUTO_TEST_CASE(ignores_leading_whitespace)
00031 {
00032 const std::vector<std::string> tokens = util::tokenize(" \t mytoken");
00033 BOOST_CHECK_EQUAL(tokens.size(), 1);
00034 BOOST_CHECK_EQUAL(tokens[0], "mytoken");
00035 }
00036
00037 BOOST_AUTO_TEST_CASE(ignores_trailing_whitespace)
00038 {
00039 const std::vector<std::string> tokens = util::tokenize("mytoken \t ");
00040 BOOST_CHECK_EQUAL(tokens.size(), 1);
00041 BOOST_CHECK_EQUAL(tokens[0], "mytoken");
00042 }
00043
00044 BOOST_AUTO_TEST_CASE(splits_tokens_on_tabs)
00045 {
00046 const std::vector<std::string> tokens = util::tokenize("one\ttwo");
00047 BOOST_CHECK_EQUAL(tokens.size(), 2);
00048 BOOST_CHECK_EQUAL(tokens[0], "one");
00049 BOOST_CHECK_EQUAL(tokens[1], "two");
00050 }
00051
00052 BOOST_AUTO_TEST_CASE(splits_tokens_on_spaces)
00053 {
00054 const std::vector<std::string> tokens = util::tokenize("one two");
00055 BOOST_CHECK_EQUAL(tokens.size(), 2);
00056 BOOST_CHECK_EQUAL(tokens[0], "one");
00057 BOOST_CHECK_EQUAL(tokens[1], "two");
00058 }
00059
00060 BOOST_AUTO_TEST_CASE(treats_sequence_of_space_as_one_space)
00061 {
00062 const std::vector<std::string> tokens = util::tokenize("one\t \ttwo");
00063 BOOST_CHECK_EQUAL(tokens.size(), 2);
00064 BOOST_CHECK_EQUAL(tokens[0], "one");
00065 BOOST_CHECK_EQUAL(tokens[1], "two");
00066 }
00067
00068 }
00069 }