Source

`stdlib/tests/parsers_json_test.cpp`

2// Original work; see ACKNOWLEDGMENTS.md for the open-source ideas we build upon.

3// In-process unit tests for the HEADER-ONLY parts of the `parsers` module: the URL parser

4// (parsers::url::Parser), the JSON schema factories (field/object), the typed struct reader

5// (read<T>() — read.hpp, which drives the shared scan.hpp scanners), the pooled construction policy

6// (PoolBuilder), the SIMD scan primitives (simd.hpp), and the JSON token classes (Boolean/Null/

7// Number/Node). The compiled DOM parser/serializer (parsers/json/json.cpp) is a direct test source

8// of this binary (see CMakeLists) and is covered separately by parsers_json_dom_test.cpp; this file

9// keeps covering the header surface directly.

11#include <cstdint>

12#include <array>

13#include <optional>

14#include <span>

15#include <string>

16#include <string_view>

17#include <tuple>

18#include <variant>

19#include <vector>

21#include <gtest/gtest.h>

23#include "json/cursor.hpp" // Cursor: the [it, end) read position the scanners advance

24#include "json/node.hpp" // Node + the Boolean/Null/Number token classes

25#include "json/pool_builder.hpp" // PoolBuilder: the pooled (viewing) construction policy

26#include "json/read.hpp" // read<T>(): parse straight into a typed struct (+ scan.hpp)

27#include "json/scan.hpp" // detail:: scanners (parse_double_fast, decode_escapes, ...)

28#include "json/schema.hpp"

29#include "json/simd.hpp" // simd:: whitespace / quote-or-backslash primitives

30#include "url/url.hpp"

32namespace json = cheatah::parsers::json;

33namespace jdet = cheatah::parsers::json::detail;

34namespace jsimd = cheatah::parsers::json::simd;

35namespace url = cheatah::parsers::url;

37namespace {

39// A struct to hang a runtime-built schema off of (mirrors how requests' Options/Response get a

40// synthesized schema<>). field()/object() are constexpr but also run at runtime.

41struct Point {

42 long long x;

43 std::string label;

44};

46// field() and object() are constexpr factories; requests uses them at compile time via schema<>.

47// Building a schema at RUNTIME here executes their bodies so they are covered.

48// cppcheck-suppress syntaxError // cppcheck mis-parses the member-pointer/decltype schema template

49TEST(CheatahParsers, SchemaFactoriesAtRuntime) {

50 volatile bool run = true; // defeat constant-folding so the factories execute at runtime

51 if (run) {

52 const auto f = json::field("x", &Point::x);

53 const auto sch = json::object(f, json::field("label", &Point::label));

54 EXPECT_EQ(std::tuple_size_v<decltype(sch.fields)>, 2u);

55 EXPECT_EQ(std::get<0>(sch.fields).name, "x");

56 EXPECT_EQ(std::get<1>(sch.fields).name, "label");

57 }

58}

60// The URL parser: scheme://host[:port][/path][?query], the exact grammar requests speaks.

61TEST(CheatahParsers, UrlParserComponents) {

62 url::Parser p;

63 url::Url u;

64 // Explicit port + path + query.

65 ASSERT_TRUE(p.parse("http://example.com:8080/a/b?x=1", u));

66 EXPECT_EQ(u.scheme, "http");

67 EXPECT_EQ(u.host, "example.com");

68 EXPECT_EQ(u.port, 8080);

69 EXPECT_EQ(u.target, "/a/b?x=1");

70 // Default port (no colon), scheme lowercased.

71 ASSERT_TRUE(p.parse("HTTP://host/path", u));

72 EXPECT_EQ(u.scheme, "http");

73 EXPECT_EQ(u.port, 80);

74 EXPECT_EQ(u.target, "/path");

75 // https default port; absent path -> "/".

76 ASSERT_TRUE(p.parse("https://secure.example", u));

77 EXPECT_EQ(u.port, 443);

78 EXPECT_EQ(u.target, "/");

79 // Bare-query form: no path, query present -> "/?...".

80 ASSERT_TRUE(p.parse("http://h?q=2", u));

81 EXPECT_EQ(u.target, "/?q=2");

82 // Max valid port.

83 ASSERT_TRUE(p.parse("http://host:65535/", u));

84 EXPECT_EQ(u.port, 65535);

85}

87TEST(CheatahParsers, UrlParserRejects) {

88 url::Parser p;

89 url::Url u;

90 EXPECT_FALSE(p.parse("no-scheme-sep", u)); // missing "://"

91 EXPECT_FALSE(p.parse("://host", u)); // empty scheme

92 EXPECT_FALSE(p.parse("ftp://host", u)); // unsupported scheme

93 EXPECT_FALSE(p.parse("http:///path", u)); // empty authority

94 EXPECT_FALSE(p.parse("http://user@host/", u)); // userinfo rejected

95 EXPECT_FALSE(p.parse("http://host/p#frag", u)); // fragment rejected

96 EXPECT_FALSE(p.parse("http://host:abc/", u)); // non-numeric port

97 EXPECT_FALSE(p.parse("http://host:0/", u)); // port too low

98 EXPECT_FALSE(p.parse("http://host:99999/", u)); // port too high

99 EXPECT_FALSE(p.parse("http://:8080/", u)); // empty host with port

100 EXPECT_FALSE(p.parse("http://host:/", u)); // empty port digits

101 EXPECT_FALSE(p.parse("http://host:123456/", u)); // >5 port digits

102}

103

104// ----------------------------------------------------------------------------

105// The header-only JSON token classes (node.hpp / number.hpp / boolean.hpp / null.hpp): construct

106// each token and read it back through value(), and exercise Node's variant() (mutable + const).

107// ----------------------------------------------------------------------------

108TEST(CheatahParsersJson, TokenClassesAndNodeVariant) {

109 // Number / Boolean / Null: constructor + value() accessor.

110 EXPECT_DOUBLE_EQ(json::Number{3.5}.value(), 3.5);

111 EXPECT_TRUE(json::Boolean{true}.value());

112 EXPECT_FALSE(json::Boolean{false}.value());

113 EXPECT_EQ(json::Null{}.value(), nullptr);

114

115 // Node wraps a token in its variant; variant() has a mutable and a const overload.

116 json::Node node{json::Number{42.0}};

117 EXPECT_DOUBLE_EQ(std::get<json::Number>(node.variant()).value(), 42.0); // mutable variant()

118 node.variant().emplace<json::Boolean>(true); // mutate via mutable ref

119 const json::Node& cref = node;

120 EXPECT_TRUE(std::get<json::Boolean>(cref.variant()).value()); // const variant()

121}

122

123// ----------------------------------------------------------------------------

124// The Array / Object / String container tokens (array.hpp / object.hpp / string.hpp): each is a

125// move-only class over its backing storage. Construct, move-construct, move-assign, read via value(),

126// and let them destruct — over BOTH the owning and viewing backings.

127// ----------------------------------------------------------------------------

128TEST(CheatahParsersJson, ContainerTokenLifecycle) {

129 // String: owning (std::string) + viewing (std::string_view). Deduction guides pick the backing.

130 json::String owned{std::string("owned")}; // String<std::string>

131 EXPECT_EQ(owned.value(), "owned");

132 json::String view{std::string_view("viewed")}; // String<std::string_view>

133 EXPECT_EQ(view.value(), "viewed");

134

135 // OwnedArray: build from a vector<Node>, move-construct and move-assign it.

136 std::vector<json::Node> elems;

137 elems.emplace_back(json::Number{1.0});

138 elems.emplace_back(json::Boolean{false});

139 json::OwnedArray arr{std::move(elems)};

140 json::OwnedArray arr2{std::move(arr)}; // move-construct

141 EXPECT_EQ(arr2.value().size(), 2u);

142 json::OwnedArray arr3{std::vector<json::Node>{}};

143 arr3 = std::move(arr2); // move-assign

144 ASSERT_EQ(arr3.value().size(), 2u);

145 EXPECT_DOUBLE_EQ(std::get<json::Number>(arr3.value()[0].variant()).value(), 1.0);

146

147 // OwnedObject: build from a vector<Member>, move-construct and move-assign it.

148 std::vector<json::Member> mem;

149 mem.push_back(json::Member{json::Node{json::String{std::string("k")}},

150 json::Node{json::Number{7.0}}});

151 json::OwnedObject obj{std::move(mem)};

152 json::OwnedObject obj2{std::move(obj)}; // move-construct

153 EXPECT_EQ(obj2.value().size(), 1u);

154 json::OwnedObject obj3{std::vector<json::Member>{}};

155 obj3 = std::move(obj2); // move-assign

156 ASSERT_EQ(obj3.value().size(), 1u);

157 EXPECT_DOUBLE_EQ(std::get<json::Number>(obj3.value()[0].second.variant()).value(), 7.0);

158}

159

160// ----------------------------------------------------------------------------

161// The SIMD scan primitives (simd.hpp): whitespace-skip (fast test + long run) and quote/backslash

162// find, over inputs long enough to exercise the AVX2 32-byte block AND its scalar tail.

163// ----------------------------------------------------------------------------

164TEST(CheatahParsersJson, SimdScanPrimitives) {

165 EXPECT_TRUE(jsimd::is_whitespace(' '));

166 EXPECT_TRUE(jsimd::is_whitespace('\t'));

167 EXPECT_FALSE(jsimd::is_whitespace('x'));

168

169 // skip_whitespace: a >40-byte run of spaces then 'X' — past the fast path, through the block scan.

170 const std::string ws = std::string(40, ' ') + "X" + std::string(5, ' ');

171 const char* p = jsimd::skip_whitespace(ws.data(), ws.data() + ws.size());

172 ASSERT_LT(p, ws.data() + ws.size());

173 EXPECT_EQ(*p, 'X');

174 // fast path: first byte already non-whitespace -> returns `it` unchanged.

175 const std::string none = "abc";

176 EXPECT_EQ(jsimd::skip_whitespace(none.data(), none.data() + none.size()), none.data());

177

178 // find_quote_or_backslash: 40 ordinary bytes then a quote, then a backslash later.

179 const std::string body = std::string(40, 'a') + "\"more\\x";

180 const char* q = jsimd::find_quote_or_backslash(body.data(), body.data() + body.size());

181 ASSERT_LT(q, body.data() + body.size());

182 EXPECT_EQ(*q, '"');

183 const char* bs = jsimd::find_quote_or_backslash(q + 1, body.data() + body.size());

184 EXPECT_EQ(*bs, '\\');

185}

186

187// ----------------------------------------------------------------------------

188// The shared low-level scanners (scan.hpp detail::): number parsing (Clinger fast path + fallbacks),

189// string scanning, and escape decoding incl. \uXXXX and a surrogate pair (append_utf8 / hex4).

190// ----------------------------------------------------------------------------

191TEST(CheatahParsersJson, ScanNumbersAndEscapes) {

192 auto parse_d = [](std::string_view s) {

193 json::Cursor c{s.data(), s.data() + s.size()};

194 double out = 0.0;

195 EXPECT_TRUE(jdet::parse_double_fast(c, out)) << s;

196 return out;

197 };

198 EXPECT_DOUBLE_EQ(parse_d("0"), 0.0);

199 EXPECT_DOUBLE_EQ(parse_d("-3"), -3.0);

200 EXPECT_DOUBLE_EQ(parse_d("2.5"), 2.5);

201 EXPECT_DOUBLE_EQ(parse_d("-12.5e2"), -1250.0);

202 EXPECT_DOUBLE_EQ(parse_d("1e-3"), 0.001);

203 // Outside the exact fast window (25 digits) -> std::from_chars fallback path.

204 EXPECT_DOUBLE_EQ(parse_d("1234567890123456789012345"), 1234567890123456789012345.0);

205

206 // parse_arithmetic on an integral T: the base-10 loop, negative, and the overflow-refetch branch.

207 auto parse_ll = [](std::string_view s) {

208 json::Cursor c{s.data(), s.data() + s.size()};

209 long long out = 0;

210 EXPECT_TRUE(jdet::parse_arithmetic(c, out)) << s;

211 return out;

212 };

213 EXPECT_EQ(parse_ll("42"), 42);

214 EXPECT_EQ(parse_ll("-9223372036854775808"), -9223372036854775807LL - 1); // INT64_MIN, refetch path

215 { // unsigned rejects a negative literal

216 const std::string neg = "-1";

217 json::Cursor c{neg.data(), neg.data() + neg.size()};

218 unsigned long long u = 0;

219 EXPECT_FALSE(jdet::parse_arithmetic(c, u));

220 }

221

222 // decode_escapes drives append_utf8 for every \uXXXX form via the 1/2/3/4-byte arms:

223 // A -> 'A' (1 byte), é -> U+00E9 (2 bytes), € -> U+20AC (3 bytes),

224 // 😀 -> U+1F600 (surrogate pair -> 4 bytes). Plus the simple two-char

225 // escapes \t \b \f \r \/ \\ \". The input is the LITERAL backslash-escape text.

226 {

227 // Build the escape text explicitly (each backslash is a real byte, not a C escape).

228 const std::string raw =

229 R"(a\tb\nA\u0041\u00e9\u20ac\ud83d\ude00\b\f\r\/\\\"z)";

230 std::string decoded;

231 ASSERT_TRUE(jdet::decode_escapes(raw, decoded));

232 EXPECT_EQ(decoded,

233 std::string("a\tb\nAA\xC3\xA9\xE2\x82\xAC\xF0\x9F\x98\x80\b\f\r/\\\"z"));

234 }

235 { // scan_string over a quoted literal with an embedded escaped quote

236 const std::string src = R"("he\"llo")";

237 json::Cursor c{src.data(), src.data() + src.size()};

238 std::string_view inner;

239 bool esc = false;

240 ASSERT_TRUE(jdet::scan_string(c, inner, esc));

241 EXPECT_TRUE(esc);

242 EXPECT_EQ(inner, R"(he\"llo)");

243 }

244 { // scan_string on a plain (escape-free) string: esc stays false, bulk path.

245 const std::string src = R"("plain")";

246 json::Cursor c{src.data(), src.data() + src.size()};

247 std::string_view inner;

248 bool esc = true;

249 ASSERT_TRUE(jdet::scan_string(c, inner, esc));

250 EXPECT_FALSE(esc);

251 EXPECT_EQ(inner, "plain");

252 }

253 { // an UNTERMINATED string (no closing quote) -> scan_string returns false.

254 const std::string src = "\"no end";

255 json::Cursor c{src.data(), src.data() + src.size()};

256 std::string_view inner;

257 bool esc = false;

258 EXPECT_FALSE(jdet::scan_string(c, inner, esc));

259 }

260 { // a dangling escape at end of input -> scan_string returns false.

261 const std::string src = "\"x\\";

262 json::Cursor c{src.data(), src.data() + src.size()};

263 std::string_view inner;

264 bool esc = false;

265 EXPECT_FALSE(jdet::scan_string(c, inner, esc));

266 }

267 // match(): the literal matcher used by skip_value's t/f/n arms — hit, miss, and too-short.

268 {

269 const std::string t = "true", n = "nullish";

270 json::Cursor ct{t.data(), t.data() + t.size()};

271 EXPECT_TRUE(jdet::match(ct, "true"));

272 json::Cursor cn{n.data(), n.data() + n.size()};

273 EXPECT_TRUE(jdet::match(cn, "null"));

274 const std::string sh = "tr";

275 json::Cursor cs{sh.data(), sh.data() + sh.size()};

276 EXPECT_FALSE(jdet::match(cs, "true")); // too short to match

277 }

278 // Malformed escapes are rejected (bad \u hex, dangling backslash, lone high surrogate, and an

279 // unknown escape selector \x -> the switch default).

280 {

281 std::string out;

282 EXPECT_FALSE(jdet::decode_escapes(R"(\uZZZZ)", out));

283 EXPECT_FALSE(jdet::decode_escapes("\\", out));

284 EXPECT_FALSE(jdet::decode_escapes(R"(\uD83Dx)", out));

285 EXPECT_FALSE(jdet::decode_escapes("\\x", out)); // unknown selector -> default: return false

286 }

287 { // an input ENDING exactly at an escape's end takes the post-loop `return true` (no npos run).

288 std::string out;

289 EXPECT_TRUE(jdet::decode_escapes("\\t", out)); // just "\t" -> a single tab, loop exits at end

290 EXPECT_EQ(out, "\t");

291 }

292 // skip_value: discard a complete nested value in one call — the object holds an array, a string,

293 // and the three literal forms true/false/null (the t/f/n match arms) plus a number.

294 {

295 const std::string src = R"({"a":[1,2,{"b":true}],"c":"x","d":false,"e":null,"f":-3.5} tail)";

296 json::Cursor c{src.data(), src.data() + src.size()};

297 ASSERT_TRUE(jdet::skip_value(c));

298 EXPECT_EQ(std::string_view(c.it, static_cast<std::size_t>(c.end - c.it)), " tail");

299 }

300 // skip_value rejects malformed shapes: a lone closing brace, and stray punctuation.

301 {

302 const std::string bad = "}";

303 json::Cursor c{bad.data(), bad.data() + bad.size()};

304 EXPECT_FALSE(jdet::skip_value(c));

305 }

306 {

307 const std::string bad = ",";

308 json::Cursor c{bad.data(), bad.data() + bad.size()};

309 EXPECT_FALSE(jdet::skip_value(c));

310 }

311 { // skip_value over just a bare literal (false) leaves the cursor at end.

312 const std::string src = "false";

313 json::Cursor c{src.data(), src.data() + src.size()};

314 EXPECT_TRUE(jdet::skip_value(c));

315 EXPECT_EQ(c.it, c.end);

316 }

317}

318

319// ----------------------------------------------------------------------------

320// PoolBuilder (pool_builder.hpp): the pooled construction stack machine. Drive it as the DOM parser

321// would — begin/add/finish for a nested array + object — and read the resulting ArrayView/ObjectView.

322// ----------------------------------------------------------------------------

323TEST(CheatahParsersJson, PoolBuilderStackMachine) {

324 json::PoolBuilder b;

325 b.reset(64); // reserve pools

326

327 // Build the array [1, "k": inner-object] -> actually: an array holding a Number then an object.

328 b.begin_array();

329 b.add_element(json::Node{json::Number{1.0}});

330 b.begin_object(); // an object nested inside the array

331 b.add_member(json::Member{json::Node{json::Number{0.0}}, json::Node{json::Boolean{true}}});

332 json::Node inner_obj = b.finish_object(); // commit_members -> ObjectView

333 b.add_element(std::move(inner_obj));

334 json::Node arr = b.finish_array(); // commit_nodes -> ArrayView

335

336 ASSERT_TRUE(std::holds_alternative<json::ArrayView>(arr.variant()));

337 const std::span<const json::Node> elems = std::get<json::ArrayView>(arr.variant()).value();

338 ASSERT_EQ(elems.size(), 2u);

339 EXPECT_DOUBLE_EQ(std::get<json::Number>(elems[0].variant()).value(), 1.0);

340 ASSERT_TRUE(std::holds_alternative<json::ObjectView>(elems[1].variant()));

341 const auto members = std::get<json::ObjectView>(elems[1].variant()).value();

342 ASSERT_EQ(members.size(), 1u);

343 EXPECT_TRUE(std::get<json::Boolean>(members[0].second.variant()).value());

344}

345

346// ----------------------------------------------------------------------------

347// The typed struct reader read<T>() (read.hpp -> detail::skip_value + integral parse_arithmetic).

348// ----------------------------------------------------------------------------

349

350struct Trade {

351 long long qty;

352 double price;

353 std::string sym;

354 std::optional<long long> lot;

355 std::vector<long long> tags;

356};

357

358// A std::array field takes read_fixed_array: exactly N elements, stored inline, never allocating.

359struct Swatch {

360 std::array<long long, 3> rgb;

361 std::string name;

362};

363

364} // namespace

365

366// schema<T> is a variable template; a struct opts in by specializing it with an object(field...)

367// description (the same non-intrusive shape requests synthesizes for its Response structs).

368namespace cheatah::parsers::json {

369template <>

370inline constexpr auto schema<Trade> = object(

371 field("qty", &Trade::qty),

372 field("price", &Trade::price),

373 field("sym", &Trade::sym),

374 field("lot", &Trade::lot),

375 field("tags", &Trade::tags));

376

377template <>

378inline constexpr auto schema<Swatch> = object(

379 field("rgb", &Swatch::rgb),

380 field("name", &Swatch::name));

381} // namespace cheatah::parsers::json

382

383namespace {

384

385// read<T>() dispatches each field on its STATIC type: integral (parse_arithmetic base-10 loop),

386// double, string, optional (null -> nullopt), vector. An UNKNOWN key exercises detail::skip_value

387// over a nested value (object containing an array/number), which must be discarded and skipped.

388TEST(CheatahParsersJson, TypedReadWithUnknownKeys) {

389 Trade t{};

390 const bool ok = json::read(

391 R"({"qty": -100, "extra": {"junk": [1, 2, {"deep": true}], "s": "skip\tme"},

392 "price": 3.25, "sym": "AAPL", "lot": null, "tags": [10, 20, 30]})",

393 t);

394 ASSERT_TRUE(ok);

395 EXPECT_EQ(t.qty, -100); // negative integral, base-10 loop

396 EXPECT_DOUBLE_EQ(t.price, 3.25);

397 EXPECT_EQ(t.sym, "AAPL");

398 EXPECT_FALSE(t.lot.has_value()); // JSON null -> nullopt

399 ASSERT_EQ(t.tags.size(), 3u);

400 EXPECT_EQ(t.tags[1], 20);

401}

402

403// A present optional and a malformed body (missing '}') that read<T> must reject.

404TEST(CheatahParsersJson, TypedReadOptionalAndReject) {

405 Trade t{};

406 ASSERT_TRUE(json::read(R"({"qty":1,"price":2,"sym":"x","lot":7,"tags":[]})", t));

407 ASSERT_TRUE(t.lot.has_value());

408 EXPECT_EQ(t.lot.value_or(-1), 7);

409

410 Trade bad{};

411 EXPECT_FALSE(json::read(R"({"qty":1,"price":2,"sym":"x","lot":null,"tags":[1)", bad));

412}

413

414// Keys in NON-schema order (exercises the hint-wrap field search), an ESCAPED string VALUE

415// (read_string -> decode_escapes), and an ESCAPED KEY (q == 'q', decode_escapes on the key).

416TEST(CheatahParsersJson, TypedReadEscapesAndKeyOrder) {

417 Trade t{};

418 // "tags" first, then "sym" with a \t escape, then the escaped key "qty" (== "qty").

419 const bool ok = json::read(

420 R"({"tags":[1,2],"sym":"a\tb","price":1.5,"\u0071ty":9,"lot":null})", t);

421 ASSERT_TRUE(ok);

422 EXPECT_EQ(t.qty, 9); // matched via the escaped key

423 EXPECT_EQ(t.sym, std::string("a\tb")); // value decoded through decode_escapes

424 ASSERT_EQ(t.tags.size(), 2u);

425 EXPECT_EQ(t.tags[0], 1);

426 EXPECT_FALSE(t.lot.has_value());

427}

428

429// A std::array field: exactly N elements fills it, and a wrong count is a parse error rather

430// than a short read — the count check is the only thing standing between the two.

431TEST(CheatahParsersJson, TypedReadFixedArray) {

432 Swatch s{};

433 ASSERT_TRUE(json::read(R"({"rgb":[12,34,56],"name":"teal"})", s));

434 EXPECT_EQ(s.rgb[0], 12);

435 EXPECT_EQ(s.rgb[2], 56);

436 EXPECT_EQ(s.name, "teal");

437

438 Swatch few{};

439 EXPECT_FALSE(json::read(R"({"rgb":[1,2],"name":"x"})", few)); // too few

440 Swatch many{};

441 EXPECT_FALSE(json::read(R"({"rgb":[1,2,3,4],"name":"x"})", many)); // too many

442}

443

444} // namespace