Source

`stdlib/tests/parsers_html_test.cpp`

2// Original work; see ACKNOWLEDGMENTS.md for the open-source ideas we build upon.

3// In-process unit tests for the `parsers.html` escaper + tolerant tokenizing parser

4// (parsers/html/html.cpp). html.cpp is compiled DIRECTLY into this test binary (see CMakeLists)

5// so its coverage is measured here — the tests below exercise escape/unescape (every entity

6// form and every malformed-reference shape), the full tokenizer (start/startend/end tags,

7// attribute quoting forms, comments/declarations/PIs, raw-text <script>/<style>, and the

8// lenient malformed-input paths), and the get_attr/has_attr helpers.

10#include <string>

11#include <vector>

13#include <gtest/gtest.h>

15#include "html/html.hpp"

17namespace html = cheatah::parsers::html;

19namespace {

21// A compact "kind|tag|data" rendering of one token, for order-sensitive assertions.

22std::string sig(const html::Token& t) { return t.kind + "|" + t.tag + "|" + t.data; }

24TEST(ParsersHtml, EscapeAllSpecialsWithAndWithoutQuote) {

25 EXPECT_EQ(html::escape(R"(<a href="x">&'z)"),

26 "<a href="x">&'z");

27 // quote=false leaves both quote characters verbatim but still escapes & < >.

28 EXPECT_EQ(html::escape(R"(<"'>&)", false), R"(<"'>&)");

29 EXPECT_EQ(html::escape(""), "");

30 EXPECT_EQ(html::escape("plain text"), "plain text"); // default arm only

31}

33TEST(ParsersHtml, UnescapeNamedAndNumericForms) {

34 // Named entities from the table.

35 EXPECT_EQ(html::unescape("<p> & "'"), "<p> & \"'");

36 EXPECT_EQ(html::unescape("©"), "\xC2\xA9");

37 EXPECT_EQ(html::unescape("€"), "\xE2\x82\xAC");

38 // Numeric decimal + hex (lower/upper x, lower/upper hex digits) across all four

39 // UTF-8 encoding widths: 1-byte A, 2-byte ©, 3-byte 中, 4-byte 😀.

40 EXPECT_EQ(html::unescape("A"), "A");

41 EXPECT_EQ(html::unescape("©"), "\xC2\xA9");

42 EXPECT_EQ(html::unescape("©"), "\xC2\xA9");

43 EXPECT_EQ(html::unescape("&#XA9;"), "\xC2\xA9");

44 EXPECT_EQ(html::unescape("中"), "\xE4\xB8\xAD");

45 EXPECT_EQ(html::unescape("😀"), "\xF0\x9F\x98\x80");

46 EXPECT_EQ(html::unescape("no refs at all"), "no refs at all");

47}

49TEST(ParsersHtml, UnescapeLeavesMalformedVerbatim) {

50 EXPECT_EQ(html::unescape("&bogus;"), "&bogus;"); // unknown name

51 EXPECT_EQ(html::unescape("a & b"), "a & b"); // bare '&', no ';'

52 EXPECT_EQ(html::unescape("&"), "&"); // '&' at end of input

53 EXPECT_EQ(html::unescape("&;"), "&;"); // empty reference body

54 EXPECT_EQ(html::unescape("&#;"), "&#;"); // numeric with no digits

55 EXPECT_EQ(html::unescape("&#x;"), "&#x;"); // hex with no digits

56 EXPECT_EQ(html::unescape("&#12z;"), "&#12z;"); // invalid decimal digit

57 EXPECT_EQ(html::unescape("&#1A;"), "&#1A;"); // hex digit in a decimal ref

58 EXPECT_EQ(html::unescape("&#x110000;"), "&#x110000;"); // beyond Unicode range

59 EXPECT_EQ(html::unescape(""), ""); // NUL rejected

60 // The ';' sits past the 32-char window, so the '&' is kept verbatim.

61 EXPECT_EQ(html::unescape("&reallyreallyreallyreallyreallylongname;"),

62 "&reallyreallyreallyreallyreallylongname;");

63 // Round-trip: escape then unescape restores the original for the covered set.

64 const std::string original = R"(<b class="x">&'</b>)";

65 EXPECT_EQ(html::unescape(html::escape(original)), original);

66}

68TEST(ParsersHtml, ParsesStartDataEndInDocumentOrder) {

69 const auto t = html::parse(R"(<div class="box">A & B</div>)");

70 ASSERT_EQ(t.size(), 3u);

71 EXPECT_EQ(sig(t[0]), "starttag|div|");

72 ASSERT_EQ(t[0].attrs.size(), 1u);

73 EXPECT_EQ(t[0].attrs[0].name, "class");

74 EXPECT_EQ(t[0].attrs[0].value, "box");

75 EXPECT_EQ(sig(t[1]), "data||A & B"); // character references decoded in text

76 EXPECT_EQ(sig(t[2]), "endtag|div|");

77}

79TEST(ParsersHtml, TagAndAttributeNamesAreLowercased) {

80 const auto t = html::parse(R"(<DIV CLASS="Keep">x</DIV>)");

81 ASSERT_EQ(t.size(), 3u);

82 EXPECT_EQ(t[0].tag, "div");

83 EXPECT_EQ(t[2].tag, "div");

84 ASSERT_EQ(t[0].attrs.size(), 1u);

85 EXPECT_EQ(t[0].attrs[0].name, "class");

86 EXPECT_EQ(t[0].attrs[0].value, "Keep"); // values keep their case

87}

89TEST(ParsersHtml, AttributeQuotingForms) {

90 const auto t = html::parse(R"(<e a="dq" b='sq' c=bare d = "spaced" disabled f="x&y">)");

91 ASSERT_EQ(t.size(), 1u);

92 ASSERT_EQ(t[0].attrs.size(), 6u);

93 EXPECT_EQ(html::get_attr(t[0], "a"), "dq");

94 EXPECT_EQ(html::get_attr(t[0], "b"), "sq");

95 EXPECT_EQ(html::get_attr(t[0], "c"), "bare"); // unquoted value

96 EXPECT_EQ(html::get_attr(t[0], "d"), "spaced"); // whitespace around '='

97 EXPECT_TRUE(html::has_attr(t[0], "disabled")); // valueless attribute

98 EXPECT_EQ(html::get_attr(t[0], "disabled"), "");

99 EXPECT_EQ(html::get_attr(t[0], "f"), "x&y"); // references decoded in values

100}

101

102TEST(ParsersHtml, AttributeEdgeCases) {

103 // A stray '=' with no name is skipped; the 'b' after it still parses as an attribute.

104 const auto stray = html::parse("<a =b>");

105 ASSERT_EQ(stray.size(), 1u);

106 ASSERT_EQ(stray[0].attrs.size(), 1u);

107 EXPECT_EQ(stray[0].attrs[0].name, "b");

108 EXPECT_EQ(stray[0].attrs[0].value, "");

109 // Unterminated quoted value: consumed to end-of-input, tag still emitted.

110 const auto unq = html::parse(R"(<a href="x)");

111 ASSERT_EQ(unq.size(), 1u);

112 EXPECT_EQ(unq[0].kind, "starttag");

113 EXPECT_EQ(html::get_attr(unq[0], "href"), "x");

114 // Unquoted value ending at end-of-input.

115 const auto bare = html::parse("<a href=x");

116 ASSERT_EQ(bare.size(), 1u);

117 EXPECT_EQ(html::get_attr(bare[0], "href"), "x");

118 // '=' with nothing after it: attribute present with an empty value.

119 const auto dangling = html::parse("<a href=");

120 ASSERT_EQ(dangling.size(), 1u);

121 EXPECT_TRUE(html::has_attr(dangling[0], "href"));

122 EXPECT_EQ(html::get_attr(dangling[0], "href"), "");

123}

124

125TEST(ParsersHtml, SelfClosingForms) {

126 const auto t = html::parse("<br/><hr /><img src='p.png'/><wbr/ ><input disabled>");

127 ASSERT_EQ(t.size(), 5u);

128 EXPECT_EQ(sig(t[0]), "startendtag|br|");

129 EXPECT_EQ(sig(t[1]), "startendtag|hr|"); // space before '/>'

130 EXPECT_EQ(sig(t[2]), "startendtag|img|");

131 EXPECT_EQ(html::get_attr(t[2], "src"), "p.png");

132 EXPECT_EQ(sig(t[3]), "startendtag|wbr|"); // space AFTER the '/': still self-closing

133 EXPECT_EQ(sig(t[4]), "starttag|input|"); // void element WITHOUT the slash: plain start

134 EXPECT_TRUE(html::has_attr(t[4], "disabled"));

135}

136

137TEST(ParsersHtml, CommentDeclarationAndPi) {

138 const auto t = html::parse("<!DOCTYPE html><?php echo 1; ?>");

139 ASSERT_EQ(t.size(), 3u);

140 EXPECT_EQ(sig(t[0]), "decl||DOCTYPE html");

141 EXPECT_EQ(sig(t[1]), "comment|| a -- comment "); // body between 

142 EXPECT_EQ(sig(t[2]), "pi||php echo 1; ?"); // body between <? and >

143 // Each construct unterminated: consumed to end-of-input, still one token.

144 const auto uc = html::parse("<!-- never closed");

145 ASSERT_EQ(uc.size(), 1u);

146 EXPECT_EQ(sig(uc[0]), "comment|| never closed");

147 const auto ud = html::parse("x<!DOCTYPE html");

148 ASSERT_EQ(ud.size(), 2u);

149 EXPECT_EQ(sig(ud[0]), "data||x"); // pending text flushed before the decl

150 EXPECT_EQ(sig(ud[1]), "decl||DOCTYPE html");

151 const auto up = html::parse("<?pi never closed");

152 ASSERT_EQ(up.size(), 1u);

153 EXPECT_EQ(sig(up[0]), "pi||pi never closed");

154 // "<!" as the very last bytes: an empty declaration, not a crash.

155 const auto bang = html::parse("y<!");

156 ASSERT_EQ(bang.size(), 2u);

157 EXPECT_EQ(sig(bang[1]), "decl||");

158}

159

160TEST(ParsersHtml, ScriptAndStyleAreRawText) {

161 // References are NOT decoded inside script; the close tag matches case-insensitively.

162 const auto t = html::parse("<SCRIPT>if (a && b < c) {}</SCRIPT><p>&</p>");

163 ASSERT_EQ(t.size(), 6u);

164 EXPECT_EQ(sig(t[0]), "starttag|script|");

165 EXPECT_EQ(sig(t[1]), "data||if (a && b < c) {}"); // verbatim body

166 EXPECT_EQ(sig(t[2]), "endtag|script|");

167 EXPECT_EQ(sig(t[4]), "data||&"); // ordinary text IS decoded

168 const auto s = html::parse("<style>a > b { color: red }</style>");

169 ASSERT_EQ(s.size(), 3u);

170 EXPECT_EQ(sig(s[1]), "data||a > b { color: red }");

171 // Empty body: no data token between the tags.

172 const auto e = html::parse("<script></script>");

173 ASSERT_EQ(e.size(), 2u);

174 EXPECT_EQ(e[0].kind, "starttag");

175 EXPECT_EQ(e[1].kind, "endtag");

176 // Unterminated: body runs to end-of-input, no close tag emitted.

177 const auto u = html::parse("<script>var a = 1;");

178 ASSERT_EQ(u.size(), 2u);

179 EXPECT_EQ(sig(u[1]), "data||var a = 1;");

180 // Close tag present but its '>' is missing: end tag still emitted, input consumed.

181 const auto nc = html::parse("<script>x</script");

182 ASSERT_EQ(nc.size(), 3u);

183 EXPECT_EQ(sig(nc[1]), "data||x");

184 EXPECT_EQ(sig(nc[2]), "endtag|script|");

185 // A SELF-CLOSED <script/> has no raw-text body: following markup parses normally.

186 const auto sc = html::parse("<script/><b>t</b>");

187 ASSERT_EQ(sc.size(), 4u);

188 EXPECT_EQ(sig(sc[0]), "startendtag|script|");

189 EXPECT_EQ(sig(sc[1]), "starttag|b|");

190}

191

192TEST(ParsersHtml, MalformedMarkupIsLenient) {

193 // Stray '<' before a non-name character is literal text.

194 const auto lt = html::parse("a < b and 3<4");

195 ASSERT_EQ(lt.size(), 1u);

196 EXPECT_EQ(sig(lt[0]), "data||a < b and 3<4");

197 // '<' as the very last byte.

198 const auto tail = html::parse("x<");

199 ASSERT_EQ(tail.size(), 1u);

200 EXPECT_EQ(sig(tail[0]), "data||x<");

201 // "</" with no name: literal text, not an end tag.

202 const auto slash = html::parse("</ div>");

203 ASSERT_EQ(slash.size(), 1u);

204 EXPECT_EQ(sig(slash[0]), "data||</ div>");

205 // "</" as the very last bytes.

206 const auto se = html::parse("y</");

207 ASSERT_EQ(se.size(), 1u);

208 EXPECT_EQ(sig(se[0]), "data||y</");

209 // An end tag whose '>' never comes: consumed to end-of-input.

210 const auto ue = html::parse("a</div");

211 ASSERT_EQ(ue.size(), 2u);

212 EXPECT_EQ(sig(ue[0]), "data||a");

213 EXPECT_EQ(sig(ue[1]), "endtag|div|");

214 // End tag with junk between the name and '>': the junk is skipped.

215 const auto junk = html::parse("</div junk>x");

216 ASSERT_EQ(junk.size(), 2u);

217 EXPECT_EQ(sig(junk[0]), "endtag|div|");

218 EXPECT_EQ(sig(junk[1]), "data||x");

219 // Unterminated start tag: attributes parsed, tag emitted, nothing after.

220 const auto us = html::parse("<div class=\"x\" id=\"y");

221 ASSERT_EQ(us.size(), 1u);

222 EXPECT_EQ(us[0].kind, "starttag");

223 EXPECT_EQ(html::get_attr(us[0], "id"), "y");

224 // Empty input: no tokens.

225 EXPECT_TRUE(html::parse("").empty());

226}

227

228TEST(ParsersHtml, NestedDocumentWalk) {

229 const auto t = html::parse("<ul id=\"m\"><li>One</li><li>Two > 1</li></ul>tail");

230 ASSERT_EQ(t.size(), 9u);

231 EXPECT_EQ(sig(t[0]), "starttag|ul|");

232 EXPECT_EQ(sig(t[1]), "starttag|li|");

233 EXPECT_EQ(sig(t[2]), "data||One");

234 EXPECT_EQ(sig(t[3]), "endtag|li|");

235 EXPECT_EQ(sig(t[4]), "starttag|li|");

236 EXPECT_EQ(sig(t[5]), "data||Two > 1");

237 EXPECT_EQ(sig(t[6]), "endtag|li|");

238 EXPECT_EQ(sig(t[7]), "endtag|ul|");

239 EXPECT_EQ(sig(t[8]), "data||tail"); // trailing text flushed at end-of-input

240}

241

242TEST(ParsersHtml, GetAttrHasAttrLookup) {

243 const auto t = html::parse(R"(<a HREF="link" data-k>x</a>)");

244 ASSERT_GE(t.size(), 1u);

245 // Case-insensitive on BOTH sides: stored names are lowercased, queries are lowered too.

246 EXPECT_EQ(html::get_attr(t[0], "href"), "link");

247 EXPECT_EQ(html::get_attr(t[0], "HREF"), "link");

248 EXPECT_TRUE(html::has_attr(t[0], "DATA-K"));

249 EXPECT_EQ(html::get_attr(t[0], "data-k"), ""); // present but valueless

250 EXPECT_FALSE(html::has_attr(t[0], "missing"));

251 EXPECT_EQ(html::get_attr(t[0], "missing"), "");

252 // A token with no attrs at all (the end tag).

253 EXPECT_FALSE(html::has_attr(t[2], "href"));

254 EXPECT_EQ(html::get_attr(t[2], "href"), "");

255}

256

257} // namespace