Source

`stdlib/tests/parsers_json_test.cpp`

2// Original work; see ACKNOWLEDGMENTS.md for the open-source ideas we build upon.

3// In-process unit tests for the HEADER-ONLY parts of the `parsers` module: the URL parser

4// (parsers::url::Parser), the JSON schema factories (field/object), the typed struct reader

5// (read<T>() — read.hpp, which drives the shared scan.hpp scanners), the pooled construction policy

6// (PoolBuilder), the SIMD scan primitives (simd.hpp), and the JSON token classes (Boolean/Null/

7// Number/Node). The compiled DOM parser/serializer (parsers/json/json.cpp) is a direct test source

8// of this binary (see CMakeLists) and is covered separately by parsers_json_dom_test.cpp; this file

9// keeps covering the header surface directly.

11#include <cstdint>

12#include <optional>

13#include <span>

14#include <string>

15#include <string_view>

16#include <tuple>

17#include <variant>

18#include <vector>

20#include <gtest/gtest.h>

22#include "json/cursor.hpp" // Cursor: the [it, end) read position the scanners advance

23#include "json/node.hpp" // Node + the Boolean/Null/Number token classes

24#include "json/pool_builder.hpp" // PoolBuilder: the pooled (viewing) construction policy

25#include "json/read.hpp" // read<T>(): parse straight into a typed struct (+ scan.hpp)

26#include "json/scan.hpp" // detail:: scanners (parse_double_fast, decode_escapes, ...)

27#include "json/schema.hpp"

28#include "json/simd.hpp" // simd:: whitespace / quote-or-backslash primitives

29#include "url/url.hpp"

31namespace json = cheatah::parsers::json;

32namespace jdet = cheatah::parsers::json::detail;

33namespace jsimd = cheatah::parsers::json::simd;

34namespace url = cheatah::parsers::url;

36namespace {

38// A struct to hang a runtime-built schema off of (mirrors how requests' Options/Response get a

39// synthesized schema<>). field()/object() are constexpr but also run at runtime.

40struct Point {

41 long long x;

42 std::string label;

43};

45// field() and object() are constexpr factories; requests uses them at compile time via schema<>.

46// Building a schema at RUNTIME here executes their bodies so they are covered.

47// cppcheck-suppress syntaxError // cppcheck mis-parses the member-pointer/decltype schema template

48TEST(CheatahParsers, SchemaFactoriesAtRuntime) {

49 volatile bool run = true; // defeat constant-folding so the factories execute at runtime

50 if (run) {

51 const auto f = json::field("x", &Point::x);

52 const auto sch = json::object(f, json::field("label", &Point::label));

53 EXPECT_EQ(std::tuple_size_v<decltype(sch.fields)>, 2u);

54 EXPECT_EQ(std::get<0>(sch.fields).name, "x");

55 EXPECT_EQ(std::get<1>(sch.fields).name, "label");

56 }

57}

59// The URL parser: scheme://host[:port][/path][?query], the exact grammar requests speaks.

60TEST(CheatahParsers, UrlParserComponents) {

61 url::Parser p;

62 url::Url u;

63 // Explicit port + path + query.

64 ASSERT_TRUE(p.parse("http://example.com:8080/a/b?x=1", u));

65 EXPECT_EQ(u.scheme, "http");

66 EXPECT_EQ(u.host, "example.com");

67 EXPECT_EQ(u.port, 8080);

68 EXPECT_EQ(u.target, "/a/b?x=1");

69 // Default port (no colon), scheme lowercased.

70 ASSERT_TRUE(p.parse("HTTP://host/path", u));

71 EXPECT_EQ(u.scheme, "http");

72 EXPECT_EQ(u.port, 80);

73 EXPECT_EQ(u.target, "/path");

74 // https default port; absent path -> "/".

75 ASSERT_TRUE(p.parse("https://secure.example", u));

76 EXPECT_EQ(u.port, 443);

77 EXPECT_EQ(u.target, "/");

78 // Bare-query form: no path, query present -> "/?...".

79 ASSERT_TRUE(p.parse("http://h?q=2", u));

80 EXPECT_EQ(u.target, "/?q=2");

81 // Max valid port.

82 ASSERT_TRUE(p.parse("http://host:65535/", u));

83 EXPECT_EQ(u.port, 65535);

84}

86TEST(CheatahParsers, UrlParserRejects) {

87 url::Parser p;

88 url::Url u;

89 EXPECT_FALSE(p.parse("no-scheme-sep", u)); // missing "://"

90 EXPECT_FALSE(p.parse("://host", u)); // empty scheme

91 EXPECT_FALSE(p.parse("ftp://host", u)); // unsupported scheme

92 EXPECT_FALSE(p.parse("http:///path", u)); // empty authority

93 EXPECT_FALSE(p.parse("http://user@host/", u)); // userinfo rejected

94 EXPECT_FALSE(p.parse("http://host/p#frag", u)); // fragment rejected

95 EXPECT_FALSE(p.parse("http://host:abc/", u)); // non-numeric port

96 EXPECT_FALSE(p.parse("http://host:0/", u)); // port too low

97 EXPECT_FALSE(p.parse("http://host:99999/", u)); // port too high

98 EXPECT_FALSE(p.parse("http://:8080/", u)); // empty host with port

99 EXPECT_FALSE(p.parse("http://host:/", u)); // empty port digits

100 EXPECT_FALSE(p.parse("http://host:123456/", u)); // >5 port digits

101}

102

103// ----------------------------------------------------------------------------

104// The header-only JSON token classes (node.hpp / number.hpp / boolean.hpp / null.hpp): construct

105// each token and read it back through value(), and exercise Node's variant() (mutable + const).

106// ----------------------------------------------------------------------------

107TEST(CheatahParsersJson, TokenClassesAndNodeVariant) {

108 // Number / Boolean / Null: constructor + value() accessor.

109 EXPECT_DOUBLE_EQ(json::Number{3.5}.value(), 3.5);

110 EXPECT_TRUE(json::Boolean{true}.value());

111 EXPECT_FALSE(json::Boolean{false}.value());

112 EXPECT_EQ(json::Null{}.value(), nullptr);

113

114 // Node wraps a token in its variant; variant() has a mutable and a const overload.

115 json::Node node{json::Number{42.0}};

116 EXPECT_DOUBLE_EQ(std::get<json::Number>(node.variant()).value(), 42.0); // mutable variant()

117 node.variant().emplace<json::Boolean>(true); // mutate via mutable ref

118 const json::Node& cref = node;

119 EXPECT_TRUE(std::get<json::Boolean>(cref.variant()).value()); // const variant()

120}

121

122// ----------------------------------------------------------------------------

123// The Array / Object / String container tokens (array.hpp / object.hpp / string.hpp): each is a

124// move-only class over its backing storage. Construct, move-construct, move-assign, read via value(),

125// and let them destruct — over BOTH the owning and viewing backings.

126// ----------------------------------------------------------------------------

127TEST(CheatahParsersJson, ContainerTokenLifecycle) {

128 // String: owning (std::string) + viewing (std::string_view). Deduction guides pick the backing.

129 json::String owned{std::string("owned")}; // String<std::string>

130 EXPECT_EQ(owned.value(), "owned");

131 json::String view{std::string_view("viewed")}; // String<std::string_view>

132 EXPECT_EQ(view.value(), "viewed");

133

134 // OwnedArray: build from a vector<Node>, move-construct and move-assign it.

135 std::vector<json::Node> elems;

136 elems.emplace_back(json::Number{1.0});

137 elems.emplace_back(json::Boolean{false});

138 json::OwnedArray arr{std::move(elems)};

139 json::OwnedArray arr2{std::move(arr)}; // move-construct

140 EXPECT_EQ(arr2.value().size(), 2u);

141 json::OwnedArray arr3{std::vector<json::Node>{}};

142 arr3 = std::move(arr2); // move-assign

143 ASSERT_EQ(arr3.value().size(), 2u);

144 EXPECT_DOUBLE_EQ(std::get<json::Number>(arr3.value()[0].variant()).value(), 1.0);

145

146 // OwnedObject: build from a vector<Member>, move-construct and move-assign it.

147 std::vector<json::Member> mem;

148 mem.push_back(json::Member{json::Node{json::String{std::string("k")}},

149 json::Node{json::Number{7.0}}});

150 json::OwnedObject obj{std::move(mem)};

151 json::OwnedObject obj2{std::move(obj)}; // move-construct

152 EXPECT_EQ(obj2.value().size(), 1u);

153 json::OwnedObject obj3{std::vector<json::Member>{}};

154 obj3 = std::move(obj2); // move-assign

155 ASSERT_EQ(obj3.value().size(), 1u);

156 EXPECT_DOUBLE_EQ(std::get<json::Number>(obj3.value()[0].second.variant()).value(), 7.0);

157}

158

159// ----------------------------------------------------------------------------

160// The SIMD scan primitives (simd.hpp): whitespace-skip (fast test + long run) and quote/backslash

161// find, over inputs long enough to exercise the AVX2 32-byte block AND its scalar tail.

162// ----------------------------------------------------------------------------

163TEST(CheatahParsersJson, SimdScanPrimitives) {

164 EXPECT_TRUE(jsimd::is_whitespace(' '));

165 EXPECT_TRUE(jsimd::is_whitespace('\t'));

166 EXPECT_FALSE(jsimd::is_whitespace('x'));

167

168 // skip_whitespace: a >40-byte run of spaces then 'X' — past the fast path, through the block scan.

169 const std::string ws = std::string(40, ' ') + "X" + std::string(5, ' ');

170 const char* p = jsimd::skip_whitespace(ws.data(), ws.data() + ws.size());

171 ASSERT_LT(p, ws.data() + ws.size());

172 EXPECT_EQ(*p, 'X');

173 // fast path: first byte already non-whitespace -> returns `it` unchanged.

174 const std::string none = "abc";

175 EXPECT_EQ(jsimd::skip_whitespace(none.data(), none.data() + none.size()), none.data());

176

177 // find_quote_or_backslash: 40 ordinary bytes then a quote, then a backslash later.

178 const std::string body = std::string(40, 'a') + "\"more\\x";

179 const char* q = jsimd::find_quote_or_backslash(body.data(), body.data() + body.size());

180 ASSERT_LT(q, body.data() + body.size());

181 EXPECT_EQ(*q, '"');

182 const char* bs = jsimd::find_quote_or_backslash(q + 1, body.data() + body.size());

183 EXPECT_EQ(*bs, '\\');

184}

185

186// ----------------------------------------------------------------------------

187// The shared low-level scanners (scan.hpp detail::): number parsing (Clinger fast path + fallbacks),

188// string scanning, and escape decoding incl. \uXXXX and a surrogate pair (append_utf8 / hex4).

189// ----------------------------------------------------------------------------

190TEST(CheatahParsersJson, ScanNumbersAndEscapes) {

191 auto parse_d = [](std::string_view s) {

192 json::Cursor c{s.data(), s.data() + s.size()};

193 double out = 0.0;

194 EXPECT_TRUE(jdet::parse_double_fast(c, out)) << s;

195 return out;

196 };

197 EXPECT_DOUBLE_EQ(parse_d("0"), 0.0);

198 EXPECT_DOUBLE_EQ(parse_d("-3"), -3.0);

199 EXPECT_DOUBLE_EQ(parse_d("2.5"), 2.5);

200 EXPECT_DOUBLE_EQ(parse_d("-12.5e2"), -1250.0);

201 EXPECT_DOUBLE_EQ(parse_d("1e-3"), 0.001);

202 // Outside the exact fast window (25 digits) -> std::from_chars fallback path.

203 EXPECT_DOUBLE_EQ(parse_d("1234567890123456789012345"), 1234567890123456789012345.0);

204

205 // parse_arithmetic on an integral T: the base-10 loop, negative, and the overflow-refetch branch.

206 auto parse_ll = [](std::string_view s) {

207 json::Cursor c{s.data(), s.data() + s.size()};

208 long long out = 0;

209 EXPECT_TRUE(jdet::parse_arithmetic(c, out)) << s;

210 return out;

211 };

212 EXPECT_EQ(parse_ll("42"), 42);

213 EXPECT_EQ(parse_ll("-9223372036854775808"), -9223372036854775807LL - 1); // INT64_MIN, refetch path

214 { // unsigned rejects a negative literal

215 const std::string neg = "-1";

216 json::Cursor c{neg.data(), neg.data() + neg.size()};

217 unsigned long long u = 0;

218 EXPECT_FALSE(jdet::parse_arithmetic(c, u));

219 }

220

221 // decode_escapes drives append_utf8 for every \uXXXX form via the 1/2/3/4-byte arms:

222 // A -> 'A' (1 byte), é -> U+00E9 (2 bytes), € -> U+20AC (3 bytes),

223 // 😀 -> U+1F600 (surrogate pair -> 4 bytes). Plus the simple two-char

224 // escapes \t \b \f \r \/ \\ \". The input is the LITERAL backslash-escape text.

225 {

226 // Build the escape text explicitly (each backslash is a real byte, not a C escape).

227 const std::string raw =

228 "a\\tb\\nA\\u0041\\u00e9\\u20ac\\ud83d\\ude00\\b\\f\\r\\/\\\\\\\"z";

229 std::string decoded;

230 ASSERT_TRUE(jdet::decode_escapes(raw, decoded));

231 EXPECT_EQ(decoded,

232 std::string("a\tb\nAA\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80\b\f\r/\\\"z"));

233 }

234 { // scan_string over a quoted literal with an embedded escaped quote

235 const std::string src = R"("he\"llo")";

236 json::Cursor c{src.data(), src.data() + src.size()};

237 std::string_view inner;

238 bool esc = false;

239 ASSERT_TRUE(jdet::scan_string(c, inner, esc));

240 EXPECT_TRUE(esc);

241 EXPECT_EQ(inner, R"(he\"llo)");

242 }

243 { // scan_string on a plain (escape-free) string: esc stays false, bulk path.

244 const std::string src = R"("plain")";

245 json::Cursor c{src.data(), src.data() + src.size()};

246 std::string_view inner;

247 bool esc = true;

248 ASSERT_TRUE(jdet::scan_string(c, inner, esc));

249 EXPECT_FALSE(esc);

250 EXPECT_EQ(inner, "plain");

251 }

252 { // an UNTERMINATED string (no closing quote) -> scan_string returns false.

253 const std::string src = "\"no end";

254 json::Cursor c{src.data(), src.data() + src.size()};

255 std::string_view inner;

256 bool esc = false;

257 EXPECT_FALSE(jdet::scan_string(c, inner, esc));

258 }

259 { // a dangling escape at end of input -> scan_string returns false.

260 const std::string src = "\"x\\";

261 json::Cursor c{src.data(), src.data() + src.size()};

262 std::string_view inner;

263 bool esc = false;

264 EXPECT_FALSE(jdet::scan_string(c, inner, esc));

265 }

266 // match(): the literal matcher used by skip_value's t/f/n arms — hit, miss, and too-short.

267 {

268 const std::string t = "true", n = "nullish";

269 json::Cursor ct{t.data(), t.data() + t.size()};

270 EXPECT_TRUE(jdet::match(ct, "true"));

271 json::Cursor cn{n.data(), n.data() + n.size()};

272 EXPECT_TRUE(jdet::match(cn, "null"));

273 const std::string sh = "tr";

274 json::Cursor cs{sh.data(), sh.data() + sh.size()};

275 EXPECT_FALSE(jdet::match(cs, "true")); // too short to match

276 }

277 // Malformed escapes are rejected (bad \u hex, dangling backslash, lone high surrogate, and an

278 // unknown escape selector \x -> the switch default).

279 {

280 std::string out;

281 EXPECT_FALSE(jdet::decode_escapes(R"(\uZZZZ)", out));

282 EXPECT_FALSE(jdet::decode_escapes("\\", out));

283 EXPECT_FALSE(jdet::decode_escapes(R"(\uD83Dx)", out));

284 EXPECT_FALSE(jdet::decode_escapes("\\x", out)); // unknown selector -> default: return false

285 }

286 { // an input ENDING exactly at an escape's end takes the post-loop `return true` (no npos run).

287 std::string out;

288 EXPECT_TRUE(jdet::decode_escapes("\\t", out)); // just "\t" -> a single tab, loop exits at end

289 EXPECT_EQ(out, "\t");

290 }

291 // skip_value: discard a complete nested value in one call — the object holds an array, a string,

292 // and the three literal forms true/false/null (the t/f/n match arms) plus a number.

293 {

294 const std::string src = R"({"a":[1,2,{"b":true}],"c":"x","d":false,"e":null,"f":-3.5} tail)";

295 json::Cursor c{src.data(), src.data() + src.size()};

296 ASSERT_TRUE(jdet::skip_value(c));

297 EXPECT_EQ(std::string_view(c.it, static_cast<std::size_t>(c.end - c.it)), " tail");

298 }

299 // skip_value rejects malformed shapes: a lone closing brace, and stray punctuation.

300 {

301 const std::string bad = "}";

302 json::Cursor c{bad.data(), bad.data() + bad.size()};

303 EXPECT_FALSE(jdet::skip_value(c));

304 }

305 {

306 const std::string bad = ",";

307 json::Cursor c{bad.data(), bad.data() + bad.size()};

308 EXPECT_FALSE(jdet::skip_value(c));

309 }

310 { // skip_value over just a bare literal (false) leaves the cursor at end.

311 const std::string src = "false";

312 json::Cursor c{src.data(), src.data() + src.size()};

313 EXPECT_TRUE(jdet::skip_value(c));

314 EXPECT_EQ(c.it, c.end);

315 }

316}

317

318// ----------------------------------------------------------------------------

319// PoolBuilder (pool_builder.hpp): the pooled construction stack machine. Drive it as the DOM parser

320// would — begin/add/finish for a nested array + object — and read the resulting ArrayView/ObjectView.

321// ----------------------------------------------------------------------------

322TEST(CheatahParsersJson, PoolBuilderStackMachine) {

323 json::PoolBuilder b;

324 b.reset(64); // reserve pools

325

326 // Build the array [1, "k": inner-object] -> actually: an array holding a Number then an object.

327 b.begin_array();

328 b.add_element(json::Node{json::Number{1.0}});

329 b.begin_object(); // an object nested inside the array

330 b.add_member(json::Member{json::Node{json::Number{0.0}}, json::Node{json::Boolean{true}}});

331 json::Node inner_obj = b.finish_object(); // commit_members -> ObjectView

332 b.add_element(std::move(inner_obj));

333 json::Node arr = b.finish_array(); // commit_nodes -> ArrayView

334

335 ASSERT_TRUE(std::holds_alternative<json::ArrayView>(arr.variant()));

336 const std::span<const json::Node> elems = std::get<json::ArrayView>(arr.variant()).value();

337 ASSERT_EQ(elems.size(), 2u);

338 EXPECT_DOUBLE_EQ(std::get<json::Number>(elems[0].variant()).value(), 1.0);

339 ASSERT_TRUE(std::holds_alternative<json::ObjectView>(elems[1].variant()));

340 const auto members = std::get<json::ObjectView>(elems[1].variant()).value();

341 ASSERT_EQ(members.size(), 1u);

342 EXPECT_TRUE(std::get<json::Boolean>(members[0].second.variant()).value());

343}

344

345// ----------------------------------------------------------------------------

346// The typed struct reader read<T>() (read.hpp -> detail::skip_value + integral parse_arithmetic).

347// ----------------------------------------------------------------------------

348

349struct Trade {

350 long long qty;

351 double price;

352 std::string sym;

353 std::optional<long long> lot;

354 std::vector<long long> tags;

355};

356

357} // namespace

358

359// schema<T> is a variable template; a struct opts in by specializing it with an object(field...)

360// description (the same non-intrusive shape requests synthesizes for its Response structs).

361namespace cheatah::parsers::json {

362template <>

363inline constexpr auto schema<Trade> = object(

364 field("qty", &Trade::qty),

365 field("price", &Trade::price),

366 field("sym", &Trade::sym),

367 field("lot", &Trade::lot),

368 field("tags", &Trade::tags));

369} // namespace cheatah::parsers::json

370

371namespace {

372

373// read<T>() dispatches each field on its STATIC type: integral (parse_arithmetic base-10 loop),

374// double, string, optional (null -> nullopt), vector. An UNKNOWN key exercises detail::skip_value

375// over a nested value (object containing an array/number), which must be discarded and skipped.

376TEST(CheatahParsersJson, TypedReadWithUnknownKeys) {

377 Trade t{};

378 const bool ok = json::read(

379 R"({"qty": -100, "extra": {"junk": [1, 2, {"deep": true}], "s": "skip\tme"},

380 "price": 3.25, "sym": "AAPL", "lot": null, "tags": [10, 20, 30]})",

381 t);

382 ASSERT_TRUE(ok);

383 EXPECT_EQ(t.qty, -100); // negative integral, base-10 loop

384 EXPECT_DOUBLE_EQ(t.price, 3.25);

385 EXPECT_EQ(t.sym, "AAPL");

386 EXPECT_FALSE(t.lot.has_value()); // JSON null -> nullopt

387 ASSERT_EQ(t.tags.size(), 3u);

388 EXPECT_EQ(t.tags[1], 20);

389}

390

391// A present optional and a malformed body (missing '}') that read<T> must reject.

392TEST(CheatahParsersJson, TypedReadOptionalAndReject) {

393 Trade t{};

394 ASSERT_TRUE(json::read(R"({"qty":1,"price":2,"sym":"x","lot":7,"tags":[]})", t));

395 ASSERT_TRUE(t.lot.has_value());

396 EXPECT_EQ(*t.lot, 7);

397

398 Trade bad{};

399 EXPECT_FALSE(json::read(R"({"qty":1,"price":2,"sym":"x","lot":null,"tags":[1)", bad));

400}

401

402// Keys in NON-schema order (exercises the hint-wrap field search), an ESCAPED string VALUE

403// (read_string -> decode_escapes), and an ESCAPED KEY (q == 'q', decode_escapes on the key).

404TEST(CheatahParsersJson, TypedReadEscapesAndKeyOrder) {

405 Trade t{};

406 // "tags" first, then "sym" with a \t escape, then the escaped key "qty" (== "qty").

407 const bool ok = json::read(

408 "{\"tags\":[1,2],\"sym\":\"a\\tb\",\"price\":1.5,\"\\u0071ty\":9,\"lot\":null}", t);

409 ASSERT_TRUE(ok);

410 EXPECT_EQ(t.qty, 9); // matched via the escaped key

411 EXPECT_EQ(t.sym, std::string("a\tb")); // value decoded through decode_escapes

412 ASSERT_EQ(t.tags.size(), 2u);

413 EXPECT_EQ(t.tags[0], 1);

414 EXPECT_FALSE(t.lot.has_value());

415}

416

417} // namespace