char-fork/llama.cpp
git clone https://git.t4t.associates/char-fork/llama.cpp
ae9afff8d
char/aliceai-t5-moe
1#include <string> 2#include <iostream> 3#include <random> 4#include <cstdlib> 5 6#include "json.h" 7#include "subproc.h" 8 9#include "jinja/runtime.h" 10#include "jinja/parser.h" 11#include "jinja/lexer.h" 12#include "jinja/utils.h" 13#include "jinja/caps.h" 14 15#include "testing.h" 16 17using json = common_json ; 18 19static void test_template (testing & t ,const std::string & name ,const std::string & tmpl ,const json & vars ,const std::string & expect ); 20 21static void test_whitespace_control (testing & t ); 22static void test_conditionals (testing & t ); 23static void test_loops (testing & t ); 24static void test_expressions (testing & t ); 25static void test_set_statement (testing & t ); 26static void test_filters (testing & t ); 27static void test_literals (testing & t ); 28static void test_comments (testing & t ); 29static void test_macros (testing & t ); 30static void test_namespace (testing & t ); 31static void test_tests (testing & t ); 32static void test_string_methods (testing & t ); 33static void test_array_methods (testing & t ); 34static void test_object_methods (testing & t ); 35static void test_hasher (testing & t ); 36static void test_stats (testing & t ); 37static void test_caps (testing & t ); 38static void test_string_parts (testing & t ); 39static void test_fuzzing (testing & t ); 40 41static bool g_python_mode = false; 42 43int main (int argc ,char * argv []) { 44testing t (std::cout ); 45t .verbose = true; 46 47// usage: test-jinja [-py] [filter_regex] 48// -py : enable python mode (use python jinja2 for rendering expected output) 49// only use this for cross-checking, not for correctness 50// note: the implementation of this flag is basic, only intented to be used by maintainers 51 52for (int i = 1 ;i < argc ;i ++ ) { 53 std::string arg = argv [i ]; 54if (arg == "-py" ) { 55g_python_mode = true; 56 }else { 57t .set_filter (arg ); 58 } 59 } 60 61t .test ("whitespace control" ,test_whitespace_control ); 62t .test ("conditionals" ,test_conditionals ); 63t .test ("loops" ,test_loops ); 64t .test ("expressions" ,test_expressions ); 65t .test ("set statement" ,test_set_statement ); 66t .test ("filters" ,test_filters ); 67t .test ("literals" ,test_literals ); 68t .test ("comments" ,test_comments ); 69t .test ("macros" ,test_macros ); 70t .test ("namespace" ,test_namespace ); 71t .test ("tests" ,test_tests ); 72t .test ("string methods" ,test_string_methods ); 73t .test ("array methods" ,test_array_methods ); 74t .test ("object methods" ,test_object_methods ); 75if (!g_python_mode ) { 76t .test ("hasher" ,test_hasher ); 77t .test ("stats" ,test_stats ); 78t .test ("caps" ,test_caps ); 79t .test ("string parts" ,test_string_parts ); 80t .test ("fuzzing" ,test_fuzzing ); 81 } 82 83return t .summary (); 84} 85 86static void test_whitespace_control (testing & t ) { 87test_template (t ,"trim_blocks removes newline after tag" , 88"{% if true %}\n" 89"hello\n" 90"{% endif %}\n" , 91 json::object (), 92"hello\n" 93 ); 94 95test_template (t ,"lstrip_blocks removes leading whitespace" , 96" {% if true %}\n" 97" hello\n" 98" {% endif %}\n" , 99 json::object (), 100" hello\n" 101 ); 102 103test_template (t ,"for loop with trim_blocks" , 104"{% for i in items %}\n" 105"{{ i }}\n" 106"{% endfor %}\n" , 107 {{"items" , json::array ({1 ,2 ,3 })}}, 108"1\n2\n3\n" 109 ); 110 111test_template (t ,"explicit strip both" , 112" {%- if true -%} \n" 113"hello\n" 114" {%- endif -%} \n" , 115 json::object (), 116"hello" 117 ); 118 119test_template (t ,"expression whitespace control" , 120" {{- 'hello' -}} \n" , 121 json::object (), 122"hello" 123 ); 124 125test_template (t ,"inline block no newline" , 126"{% if true %}yes{% endif %}" , 127 json::object (), 128"yes" 129 ); 130} 131 132static void test_conditionals (testing & t ) { 133test_template (t ,"if true" , 134"{% if cond %}yes{% endif %}" , 135 {{"cond" , true}}, 136"yes" 137 ); 138 139test_template (t ,"if false" , 140"{% if cond %}yes{% endif %}" , 141 {{"cond" , false}}, 142"" 143 ); 144 145test_template (t ,"if else" , 146"{% if cond %}yes{% else %}no{% endif %}" , 147 {{"cond" , false}}, 148"no" 149 ); 150 151test_template (t ,"if elif else" , 152"{% if a %}A{% elif b %}B{% else %}C{% endif %}" , 153 {{"a" , false}, {"b" , true}}, 154"B" 155 ); 156 157test_template (t ,"nested if" , 158"{% if outer %}{% if inner %}both{% endif %}{% endif %}" , 159 {{"outer" , true}, {"inner" , true}}, 160"both" 161 ); 162 163test_template (t ,"comparison operators" , 164"{% if x > 5 %}big{% endif %}" , 165 {{"x" ,10 }}, 166"big" 167 ); 168 169test_template (t ,"object comparison" , 170"{% if {0: 1, none: 2, 1.0: 3, '0': 4, true: 5} == {false: 1, none: 2, 1: 5, '0': 4} %}equal{% endif %}" , 171 json::object (), 172"equal" 173 ); 174 175test_template (t ,"array comparison" , 176"{% if [0, 1.0, false] == [false, 1, 0.0] %}equal{% endif %}" , 177 json::object (), 178"equal" 179 ); 180 181test_template (t ,"logical and" , 182"{% if a and b %}both{% endif %}" , 183 {{"a" , true}, {"b" , true}}, 184"both" 185 ); 186 187test_template (t ,"logical or" , 188"{% if a or b %}either{% endif %}" , 189 {{"a" , false}, {"b" , true}}, 190"either" 191 ); 192 193test_template (t ,"logical not" , 194"{% if not a %}negated{% endif %}" , 195 {{"a" , false}}, 196"negated" 197 ); 198 199test_template (t ,"in operator (element in array)" , 200"{% if 'x' in items %}found{% endif %}" , 201 {{"items" , json::array ({"x" ,"y" })}}, 202"found" 203 ); 204 205test_template (t ,"in operator (substring)" , 206"{% if 'bc' in 'abcd' %}found{% endif %}" , 207 json::object (), 208"found" 209 ); 210 211test_template (t ,"in operator (object key)" , 212"{% if 'key' in obj %}found{% endif %}" , 213 {{"obj" , {{"key" ,1 }, {"other" ,2 }}}}, 214"found" 215 ); 216 217test_template (t ,"is defined" , 218"{% if x is defined %}yes{% else %}no{% endif %}" , 219 {{"x" ,1 }}, 220"yes" 221 ); 222 223test_template (t ,"is not defined" , 224"{% if y is not defined %}yes{% else %}no{% endif %}" , 225 json::object (), 226"yes" 227 ); 228 229test_template (t ,"is undefined falsy" , 230"{{ 'yes' if not y else 'no' }}" , 231 json::object (), 232"yes" 233 ); 234 235test_template (t ,"is undefined attribute falsy" , 236"{{ 'yes' if not y.x else 'no' }}" , 237 {{"y" , true}}, 238"yes" 239 ); 240 241test_template (t ,"is undefined key falsy" , 242"{{ 'yes' if not y['x'] else 'no' }}" , 243 {{"y" , json::array ({nullptr })}}, 244"yes" 245 ); 246 247test_template (t ,"is empty array falsy" , 248"{{ 'yes' if not y else 'no' }}" , 249 {{"y" , json::array ()}}, 250"yes" 251 ); 252 253test_template (t ,"is empty object falsy" , 254"{{ 'yes' if not y else 'no' }}" , 255 {{"y" , json::object ()}}, 256"yes" 257 ); 258 259test_template (t ,"is empty string falsy" , 260"{{ 'yes' if not y else 'no' }}" , 261 {{"y" ,"" }}, 262"yes" 263 ); 264 265test_template (t ,"is 0 falsy" , 266"{{ 'yes' if not y else 'no' }}" , 267 {{"y" ,0 }}, 268"yes" 269 ); 270 271test_template (t ,"is 0.0 falsy" , 272"{{ 'yes' if not y else 'no' }}" , 273 {{"y" ,0.0 }}, 274"yes" 275 ); 276 277test_template (t ,"is non-empty array truthy" , 278"{{ 'yes' if y else 'no' }}" , 279 {{"y" , json::array ({"" })}}, 280"yes" 281 ); 282 283test_template (t ,"is non-empty object truthy" , 284"{{ 'yes' if y else 'no' }}" , 285 {{"y" , json::array ({"x" , false})}}, 286"yes" 287 ); 288 289test_template (t ,"is non-empty string truthy" , 290"{{ 'yes' if y else 'no' }}" , 291 {{"y" ,"0" }}, 292"yes" 293 ); 294 295test_template (t ,"is 1 truthy" , 296"{{ 'yes' if y else 'no' }}" , 297 {{"y" ,1 }}, 298"yes" 299 ); 300 301test_template (t ,"is 1.0 truthy" , 302"{{ 'yes' if y else 'no' }}" , 303 {{"y" ,1.0 }}, 304"yes" 305 ); 306} 307 308static void test_loops (testing & t ) { 309test_template (t ,"simple for" , 310"{% for i in items %}{{ i }}{% endfor %}" , 311 {{"items" , json::array ({1 ,2 ,3 })}}, 312"123" 313 ); 314 315test_template (t ,"loop.index" , 316"{% for i in items %}{{ loop.index }}{% endfor %}" , 317 {{"items" , json::array ({"a" ,"b" ,"c" })}}, 318"123" 319 ); 320 321test_template (t ,"loop.index0" , 322"{% for i in items %}{{ loop.index0 }}{% endfor %}" , 323 {{"items" , json::array ({"a" ,"b" ,"c" })}}, 324"012" 325 ); 326 327test_template (t ,"loop.first and loop.last" , 328"{% for i in items %}{% if loop.first %}[{% endif %}{{ i }}{% if loop.last %}]{% endif %}{% endfor %}" , 329 {{"items" , json::array ({1 ,2 ,3 })}}, 330"[123]" 331 ); 332 333test_template (t ,"loop.length" , 334"{% for i in items %}{{ loop.length }}{% endfor %}" , 335 {{"items" , json::array ({"a" ,"b" })}}, 336"22" 337 ); 338 339test_template (t ,"for over dict items" , 340"{% for k, v in data.items() %}{{ k }}={{ v }} {% endfor %}" , 341 {{"data" , {{"x" ,1 }, {"y" ,2 }}}}, 342"x=1 y=2 " 343 ); 344 345test_template (t ,"for else empty" , 346"{% for i in items %}{{ i }}{% else %}empty{% endfor %}" , 347 {{"items" , json::array ()}}, 348"empty" 349 ); 350 351test_template (t ,"for undefined empty" , 352"{% for i in items %}{{ i }}{% else %}empty{% endfor %}" , 353 json::object (), 354"empty" 355 ); 356 357test_template (t ,"nested for" , 358"{% for i in a %}{% for j in b %}{{ i }}{{ j }}{% endfor %}{% endfor %}" , 359 {{"a" , json::array ({1 ,2 })}, {"b" , json::array ({"x" ,"y" })}}, 360"1x1y2x2y" 361 ); 362 363test_template (t ,"for with range" , 364"{% for i in range(3) %}{{ i }}{% endfor %}" , 365 json::object (), 366"012" 367 ); 368} 369 370static void test_expressions (testing & t ) { 371test_template (t ,"simple variable" , 372"{{ x }}" , 373 {{"x" ,42 }}, 374"42" 375 ); 376 377test_template (t ,"none in object" , 378"{{ x in {'low': 1, 'high': 2} }}" , 379 {{"x" ,nullptr }}, 380"False" 381 ); 382 383test_template (t ,"none not in object" , 384"{{ x not in {'low': 1, 'high': 2} }}" , 385 {{"x" ,nullptr }}, 386"True" 387 ); 388 389test_template (t ,"none in array" , 390"{{ x in [1, none, 3] }}" , 391 {{"x" ,nullptr }}, 392"True" 393 ); 394 395test_template (t ,"dot notation" , 396"{{ user.name }}" , 397 {{"user" , {{"name" ,"Bob" }}}}, 398"Bob" 399 ); 400 401test_template (t ,"dot notation (integer property)" , 402"{{ {10: 'Bob'}.10 }}" , 403 json::object (), 404"Bob" 405 ); 406 407test_template (t ,"dot notation (array index)" , 408"{{ user.10 }}" , 409 {{"user" , json::array ({"a" ,"b" ,"c" ,"d" ,"e" ,"f" ,"g" ,"h" ,"i" ,"j" ,"k" })}}, 410"k" 411 ); 412 413test_template (t ,"negative float (not dot notation)" , 414"{{ -1.0 }}" , 415 json::object (), 416"-1.0" 417 ); 418 419test_template (t ,"bracket notation" , 420"{{ user['name'] }}" , 421 {{"user" , {{"name" ,"Bob" }}}}, 422"Bob" 423 ); 424 425test_template (t ,"empty computed member defaults to undefined" , 426"{{ a[]|default('fallback') }}" , 427 {{"a" , {{"name" ,"Bob" }}}}, 428"fallback" 429 ); 430 431test_template (t ,"empty computed member is undefined" , 432"{{ a[] is undefined }}" , 433 {{"a" , {{"name" ,"Bob" }}}}, 434"True" 435 ); 436 437test_template (t ,"undefined computed member is undefined" , 438"{{ a[undefined] is undefined }}" , 439 {{"a" , {{"name" ,"Bob" }}}}, 440"True" 441 ); 442 443test_template (t ,"array access" , 444"{{ items[1] }}" , 445 {{"items" , json::array ({"a" ,"b" ,"c" })}}, 446"b" 447 ); 448 449test_template (t ,"array negative access" , 450"{{ items[-1] }}" , 451 {{"items" , json::array ({"a" ,"b" ,"c" })}}, 452"c" 453 ); 454 455test_template (t ,"array slice" , 456"{{ items[1:-1]|string }}" , 457 {{"items" , json::array ({"a" ,"b" ,"c" })}}, 458"['b']" 459 ); 460 461test_template (t ,"array slice step" , 462"{{ items[::2]|string }}" , 463 {{"items" , json::array ({"a" ,"b" ,"c" })}}, 464"['a', 'c']" 465 ); 466 467test_template (t ,"tuple slice" , 468"{{ ('a', 'b', 'c')[::-1]|string }}" , 469 json::object (), 470"('c', 'b', 'a')" 471 ); 472 473test_template (t ,"string slice negative step" , 474"{{ 'abcdef'[::-2] }}" , 475 json::object (), 476"fdb" 477 ); 478 479test_template (t ,"string slice negative start and step" , 480"{{ 'abcdef'[-1:1:-1] }}" , 481 json::object (), 482"fedc" 483 ); 484 485test_template (t ,"string slice negative start, stop and step" , 486"{{ 'abcdef'[-1:-5:-1] }}" , 487 json::object (), 488"fedc" 489 ); 490 491test_template (t ,"arithmetic" , 492"{{ (a + b) * c }}" , 493 {{"a" ,2 }, {"b" ,3 }, {"c" ,4 }}, 494"20" 495 ); 496 497test_template (t ,"string concat ~" , 498"{{ 'hello' ~ ' ' ~ 'world' }}" , 499 json::object (), 500"hello world" 501 ); 502 503test_template (t ,"string repetition" , 504"{{ 'ab' * 3 }}" , 505 json::object (), 506"ababab" 507 ); 508 509test_template (t ,"reversed string repetition" , 510"{{ 3 * 'ab' }}" , 511 json::object (), 512"ababab" 513 ); 514 515test_template (t ,"ternary" , 516"{{ 'yes' if cond else 'no' }}" , 517 {{"cond" , true}}, 518"yes" 519 ); 520} 521 522static void test_set_statement (testing & t ) { 523test_template (t ,"simple set" , 524"{% set x = 5 %}{{ x }}" , 525 json::object (), 526"5" 527 ); 528 529test_template (t ,"set with expression" , 530"{% set x = a + b %}{{ x }}" , 531 {{"a" ,10 }, {"b" ,20 }}, 532"30" 533 ); 534 535test_template (t ,"set list" , 536"{% set items = [1, 2, 3] %}{{ items|length }}" , 537 json::object (), 538"3" 539 ); 540 541test_template (t ,"set dict" , 542"{% set d = {'a': 1} %}{{ d.a }}" , 543 json::object (), 544"1" 545 ); 546 547test_template (t ,"set dict with mixed type keys" , 548"{% set d = {0: 1, none: 2, 1.0: 3, '0': 4, (0, 0): 5, false: 6, 1: 7} %}{{ d[(0, 0)] + d[0] + d[none] + d['0'] + d[false] + d[1.0] + d[1] }}" , 549 json::object (), 550"37" 551 ); 552 553test_template (t ,"print dict with mixed type keys" , 554"{% set d = {0: 1, none: 2, 1.0: 3, '0': 4, (0, 0): 5, true: 6} %}{{ d|string }}" , 555 json::object (), 556"{0: 1, None: 2, 1.0: 6, '0': 4, (0, 0): 5}" 557 ); 558 559test_template (t ,"print array with mixed types" , 560"{% set d = [0, none, 1.0, '0', true, (0, 0)] %}{{ d|string }}" , 561 json::object (), 562"[0, None, 1.0, '0', True, (0, 0)]" 563 ); 564 565test_template (t ,"object member assignment with mixed key types" , 566"{% set d = namespace() %}{% set d.a = 123 %}{{ d['a'] == 123 }}" , 567 json::object (), 568"True" 569 ); 570 571test_template (t ,"tuple unpacking" , 572"{% set t = (1, 2, 3) %}{% set a, b, c = t %}{{ a + b + c }}" , 573 json::object (), 574"6" 575 ); 576} 577 578static void test_filters (testing & t ) { 579test_template (t ,"upper" , 580"{{ 'hello'|upper }}" , 581 json::object (), 582"HELLO" 583 ); 584 585test_template (t ,"lower" , 586"{{ 'HELLO'|lower }}" , 587 json::object (), 588"hello" 589 ); 590 591test_template (t ,"upper array" , 592"{{ items|upper }}" , 593 {{"items" , json::array ({"hello" ,"world" })}}, 594"['HELLO', 'WORLD']" 595 ); 596 597test_template (t ,"upper dict" , 598"{{ items|upper }}" , 599 {{"items" , {{"hello" ,"world" }}}}, 600"{'HELLO': 'WORLD'}" 601 ); 602 603test_template (t ,"capitalize" , 604"{{ 'heLlo World'|capitalize }}" , 605 json::object (), 606"Hello world" 607 ); 608 609test_template (t ,"title" , 610"{{ 'hello world'|title }}" , 611 json::object (), 612"Hello World" 613 ); 614 615test_template (t ,"trim" , 616"{{ ' \r\n\thello\t\n\r '|trim }}" , 617 json::object (), 618"hello" 619 ); 620 621test_template (t ,"trim chars" , 622"{{ 'xyxhelloxyx'|trim('xy') }}" , 623 json::object (), 624"hello" 625 ); 626 627test_template (t ,"length string" , 628"{{ 'hello'|length }}" , 629 json::object (), 630"5" 631 ); 632 633test_template (t ,"replace" , 634"{{ 'hello world'|replace('world', 'jinja') }}" , 635 json::object (), 636"hello jinja" 637 ); 638 639test_template (t ,"length (count alias) list" , 640"{{ items|count }}" , 641 {{"items" , json::array ({1 ,2 ,3 })}}, 642"3" 643 ); 644 645test_template (t ,"first" , 646"{{ items|first }}" , 647 {{"items" , json::array ({10 ,20 ,30 })}}, 648"10" 649 ); 650 651test_template (t ,"last" , 652"{{ items|last }}" , 653 {{"items" , json::array ({10 ,20 ,30 })}}, 654"30" 655 ); 656 657test_template (t ,"reverse" , 658"{% for i in items|reverse %}{{ i }}{% endfor %}" , 659 {{"items" , json::array ({1 ,2 ,3 })}}, 660"321" 661 ); 662 663test_template (t ,"sort" , 664"{% for i in items|sort %}{{ i }}{% endfor %}" , 665 {{"items" , json::array ({3 ,1 ,2 })}}, 666"123" 667 ); 668 669test_template (t ,"sort reverse" , 670"{% for i in items|sort(true) %}{{ i }}{% endfor %}" , 671 {{"items" , json::array ({3 ,1 ,2 })}}, 672"321" 673 ); 674 675test_template (t ,"sort with attribute" , 676"{{ items|sort(attribute='name')|join(attribute='age') }}" , 677 {{"items" , json::array ({ 678json ({{"name" ,"c" }, {"age" ,3 }}), 679json ({{"name" ,"a" }, {"age" ,1 }}), 680json ({{"name" ,"b" }, {"age" ,2 }}), 681 })}}, 682"123" 683 ); 684 685test_template (t ,"sort with numeric attribute" , 686"{{ items|sort(attribute=0)|join(attribute=1) }}" , 687 {{"items" , json::array ({ 688 json::array ({3 ,"z" }), 689 json::array ({1 ,"x" }), 690 json::array ({2 ,"y" }), 691 })}}, 692"xyz" 693 ); 694 695test_template (t ,"join" , 696"{{ items|join(', ') }}" , 697 {{"items" , json::array ({"a" ,"b" ,"c" })}}, 698"a, b, c" 699 ); 700 701test_template (t ,"join default separator" , 702"{{ items|join }}" , 703 {{"items" , json::array ({"x" ,"y" ,"z" })}}, 704"xyz" 705 ); 706 707test_template (t ,"abs" , 708"{{ -5|abs }}" , 709 json::object (), 710"5" 711 ); 712 713test_template (t ,"int from string" , 714"{{ '42'|int }}" , 715 json::object (), 716"42" 717 ); 718 719test_template (t ,"int from string with default" , 720"{{ ''|int(1) }}" , 721 json::object (), 722"1" 723 ); 724 725test_template (t ,"int from string with base" , 726"{{ '11'|int(base=2) }}" , 727 json::object (), 728"3" 729 ); 730 731test_template (t ,"float from string" , 732"{{ '3.14'|float }}" , 733 json::object (), 734"3.14" 735 ); 736 737test_template (t ,"default with value" , 738"{{ x|default('fallback') }}" , 739 {{"x" ,"actual" }}, 740"actual" 741 ); 742 743test_template (t ,"default without value" , 744"{{ y|default('fallback') }}" , 745 json::object (), 746"fallback" 747 ); 748 749test_template (t ,"default (d alias) with falsy value" , 750"{{ ''|d('fallback', true) }}" , 751 json::object (), 752"fallback" 753 ); 754 755test_template (t ,"tojson ensure_ascii=true" , 756"{{ data|tojson(ensure_ascii=true) }}" , 757 {{"data" ,"\u2713" }}, 758"\"\\u2713\"" 759 ); 760 761test_template (t ,"tojson ensure_ascii=true nested object" , 762"{{ data|tojson(ensure_ascii=true) }}" , 763 {{"data" , { 764 {"text" ,"\u2713" }, 765 {"items" , json::array ({"é" , {{"snowman" ,"☃" }}})} 766 }}}, 767"{\"text\": \"\\u2713\", \"items\": [\"\\u00e9\", {\"snowman\": \"\\u2603\"}]}" 768 ); 769 770test_template (t ,"tojson ensure_ascii=true indent=2" , 771"{{ data|tojson(ensure_ascii=true, indent=2) }}" , 772 {{"data" , { 773 {"text" ,"\u2713" }, 774 {"nested" , {{"accent" ,"é" }}} 775 }}}, 776"{\n \"text\": \"\\u2713\",\n \"nested\": {\n \"accent\": \"\\u00e9\"\n }\n}" 777 ); 778 779test_template (t ,"tojson ensure_ascii=true preserves existing escapes" , 780"{{ data|tojson(ensure_ascii=true) }}" , 781 {{"data" , { 782 {"emoji" ,"😀" }, 783 {"line" ,"a\nb" } 784 }}}, 785"{\"emoji\": \"\\ud83d\\ude00\", \"line\": \"a\\nb\"}" 786 ); 787 788test_template (t ,"tojson sort_keys=true" , 789"{{ data|tojson(sort_keys=true) }}" , 790 {{"data" , {{"b" ,2 }, {"a" ,1 }}}}, 791"{\"a\": 1, \"b\": 2}" 792 ); 793 794test_template (t ,"tojson" , 795"{{ data|tojson }}" , 796 {{"data" , {{"a" ,1 }, {"b" , json::array ({1 ,2 })}}}}, 797"{\"a\": 1, \"b\": [1, 2]}" 798 ); 799 800test_template (t ,"tojson indent=4" , 801"{{ data|tojson(indent=4) }}" , 802 {{"data" , {{"a" ,1 }, {"b" , json::array ({1 ,2 })}}}}, 803"{\n \"a\": 1,\n \"b\": [\n 1,\n 2\n ]\n}" 804 ); 805 806test_template (t ,"tojson separators=(',',':')" , 807"{{ data|tojson(separators=(',',':')) }}" , 808 {{"data" , {{"a" ,1 }, {"b" , json::array ({1 ,2 })}}}}, 809"{\"a\":1,\"b\":[1,2]}" 810 ); 811 812test_template (t ,"tojson separators=(',',': ') indent=2" , 813"{{ data|tojson(separators=(',',': '), indent=2) }}" , 814 {{"data" , {{"a" ,1 }, {"b" , json::array ({1 ,2 })}}}}, 815"{\n \"a\": 1,\n \"b\": [\n 1,\n 2\n ]\n}" 816 ); 817 818test_template (t ,"indent" , 819"{{ data|indent(2) }}" , 820 {{"data" ,"foo\nbar" }}, 821"foo\n bar" 822 ); 823 824test_template (t ,"indent first only" , 825"{{ data|indent(width=3,first=true) }}" , 826 {{"data" ,"foo\nbar" }}, 827" foo\n bar" 828 ); 829 830test_template (t ,"indent blank lines and first line" , 831"{{ data|indent(width=5,blank=true,first=true) }}" , 832 {{"data" ,"foo\n\nbar" }}, 833" foo\n \n bar" 834 ); 835 836test_template (t ,"indent with default width" , 837"{{ data|indent() }}" , 838 {{"data" ,"foo\nbar" }}, 839"foo\n bar" 840 ); 841 842test_template (t ,"indent with no newline" , 843"{{ data|indent }}" , 844 {{"data" ,"foo" }}, 845"foo" 846 ); 847 848test_template (t ,"indent with trailing newline" , 849"{{ data|indent(blank=true) }}" , 850 {{"data" ,"foo\n" }}, 851"foo\n " 852 ); 853 854test_template (t ,"indent with string" , 855"{{ data|indent(width='>>>>') }}" , 856 {{"data" ,"foo\nbar" }}, 857"foo\n>>>>bar" 858 ); 859 860test_template (t ,"chained filters" , 861"{{ ' HELLO '|trim|lower }}" , 862 json::object (), 863"hello" 864 ); 865 866test_template (t ,"int filter on integer is identity" , 867"{{ value|int }}" , 868 {{"value" ,7 }}, 869"7" 870 ); 871 872test_template (t ,"none to string" , 873"{{ x|string }}" , 874 {{"x" ,nullptr }}, 875"None" 876 ); 877} 878 879static void test_literals (testing & t ) { 880test_template (t ,"integer" , 881"{{ 42 }}" , 882 json::object (), 883"42" 884 ); 885 886test_template (t ,"float" , 887"{{ 3.14 }}" , 888 json::object (), 889"3.14" 890 ); 891 892test_template (t ,"string" , 893"{{ 'hello' }}" , 894 json::object (), 895"hello" 896 ); 897 898test_template (t ,"boolean true" , 899"{{ true }}" , 900 json::object (), 901"True" 902 ); 903 904test_template (t ,"boolean false" , 905"{{ false }}" , 906 json::object (), 907"False" 908 ); 909 910test_template (t ,"none" , 911"{% if x is none %}null{% endif %}" , 912 {{"x" ,nullptr }}, 913"null" 914 ); 915 916test_template (t ,"list literal" , 917"{% for i in [1, 2, 3] %}{{ i }}{% endfor %}" , 918 json::object (), 919"123" 920 ); 921 922test_template (t ,"dict literal" , 923"{% set d = {'a': 1} %}{{ d.a }}" , 924 json::object (), 925"1" 926 ); 927 928test_template (t ,"integer|abs" , 929"{{ -42 | abs }}" , 930 json::object (), 931"42" 932 ); 933 934test_template (t ,"integer|float" , 935"{{ 42 | float }}" , 936 json::object (), 937"42.0" 938 ); 939 940test_template (t ,"integer|tojson" , 941"{{ 42 | tojson }}" , 942 json::object (), 943"42" 944 ); 945 946test_template (t ,"float|abs" , 947"{{ -3.14 | abs }}" , 948 json::object (), 949"3.14" 950 ); 951 952test_template (t ,"float|int" , 953"{{ 3.14 | int }}" , 954 json::object (), 955"3" 956 ); 957 958test_template (t ,"float|tojson" , 959"{{ 3.14 | tojson }}" , 960 json::object (), 961"3.14" 962 ); 963 964test_template (t ,"string|tojson" , 965"{{ 'hello' | tojson }}" , 966 json::object (), 967"\"hello\"" 968 ); 969 970test_template (t ,"boolean|int" , 971"{{ true | int }}" , 972 json::object (), 973"1" 974 ); 975 976test_template (t ,"boolean|float" , 977"{{ true | float }}" , 978 json::object (), 979"1.0" 980 ); 981 982test_template (t ,"boolean|tojson" , 983"{{ true | tojson }}" , 984 json::object (), 985"true" 986 ); 987} 988 989static void test_comments (testing & t ) { 990test_template (t ,"inline comment" , 991"before{# comment #}after" , 992 json::object (), 993"beforeafter" 994 ); 995 996test_template (t ,"comment ignores code" , 997"{% set x = 1 %}{# {% set x = 999 %} #}{{ x }}" , 998 json::object (), 999"1" 1000 ); 1001} 1002 1003static void test_macros (testing & t ) { 1004test_template (t ,"simple macro" , 1005"{% macro greet(name) %}Hello {{ name }}{% endmacro %}{{ greet('World') }}" , 1006 json::object (), 1007"Hello World" 1008 ); 1009 1010test_template (t ,"macro default arg" , 1011"{% macro greet(name='Guest') %}Hi {{ name }}{% endmacro %}{{ greet() }}" , 1012 json::object (), 1013"Hi Guest" 1014 ); 1015 1016test_template (t ,"macro kwargs input" , 1017"{% macro my_func(a, b=False) %}{% if b %}{{ a }}{% else %}nope{% endif %}{% endmacro %}{{ my_func(1, b=True) }}" , 1018 json::object (), 1019"1" 1020 ); 1021 1022test_template (t ,"macro with multiple args" , 1023"{% macro add(a, b, c=0) %}{{ a + b + c }}{% endmacro %}{{ add(1, 2) }},{{ add(1, 2, 3) }},{{ add(1, b=10) }},{{ add(1, 2, c=5) }}" , 1024 json::object (), 1025"3,6,11,8" 1026 ); 1027 1028test_template (t ,"macro with kwarg out-of-order input" , 1029"{% macro greet(first, last, greeting='Hello') %}{{ greeting }}, {{ first }} {{ last }}{% endmacro %}{{ greet(last='Smith', first='John') }},{{ greet(last='Doe', greeting='Hi', first='Jane') }}" , 1030 json::object (), 1031"Hello, John Smith,Hi, Jane Doe" 1032 ); 1033 1034test_template (t ,"macro with caller" , 1035"\ 1036{%- macro nest_dict(o, i, ff='') %}\n\ 1037{{- caller(ff) }}\n\ 1038{%- for k, v in o|items %}\n\ 1039{{- i + k + ': ' }}\n\ 1040{%- if v is mapping %}\n\ 1041{{- '{' }}\n\ 1042{% call(f) nest_dict(v, i + ' ') %}\n\ 1043{{- 'fail' if ff is undefined }}\n\ 1044{%- endcall %}\n\ 1045{{- i + '}' }}\n\ 1046{% else %}\n\ 1047{{- v|string }}\n\ 1048{% endif %}\n\ 1049{%- endfor %}\n\ 1050{%- endmacro %}\n\ 1051{%- call(f) nest_dict({'root1': 1, 'root2': {'nest1': 1, 'nest2': {'nest3': 2}}}, ' ', 'Dict') %}\n\ 1052{{- 'fail' if ff is defined }}\n\ 1053{{- f + ' {' }}\n\ 1054{% endcall %}\n\ 1055{{- '}' }}" , 1056 json::object (), 1057"Dict {\n root1: 1\n root2: {\n nest1: 1\n nest2: {\n nest3: 2\n }\n }\n}" 1058 ); 1059} 1060 1061static void test_namespace (testing & t ) { 1062test_template (t ,"namespace counter" , 1063"{% set ns = namespace(count=0) %}{% for i in range(3) %}{% set ns.count = ns.count + 1 %}{% endfor %}{{ ns.count }}" , 1064 json::object (), 1065"3" 1066 ); 1067} 1068 1069static void test_tests (testing & t ) { 1070test_template (t ,"is odd" , 1071"{% if 3 is odd %}yes{% endif %}" , 1072 json::object (), 1073"yes" 1074 ); 1075 1076test_template (t ,"is even" , 1077"{% if 4 is even %}yes{% endif %}" , 1078 json::object (), 1079"yes" 1080 ); 1081 1082test_template (t ,"is false" , 1083"{{ 'yes' if x is false }}" , 1084 {{"x" , false}}, 1085"yes" 1086 ); 1087 1088test_template (t ,"is true" , 1089"{{ 'yes' if x is true }}" , 1090 {{"x" , true}}, 1091"yes" 1092 ); 1093 1094test_template (t ,"string is false" , 1095"{{ 'yes' if x is false else 'no' }}" , 1096 {{"x" ,"" }}, 1097"no" 1098 ); 1099 1100test_template (t ,"is divisibleby" , 1101"{{ 'yes' if x is divisibleby(2) }}" , 1102 {{"x" ,2 }}, 1103"yes" 1104 ); 1105 1106test_template (t ,"is eq" , 1107"{{ 'yes' if 3 is eq(3) }}" , 1108 json::object (), 1109"yes" 1110 ); 1111 1112test_template (t ,"is not equalto" , 1113"{{ 'yes' if 3 is not equalto(4) }}" , 1114 json::object (), 1115"yes" 1116 ); 1117 1118test_template (t ,"is ge" , 1119"{{ 'yes' if 3 is ge(3) }}" , 1120 json::object (), 1121"yes" 1122 ); 1123 1124test_template (t ,"is gt" , 1125"{{ 'yes' if 3 is gt(2) }}" , 1126 json::object (), 1127"yes" 1128 ); 1129 1130test_template (t ,"is greaterthan" , 1131"{{ 'yes' if 3 is greaterthan(2) }}" , 1132 json::object (), 1133"yes" 1134 ); 1135 1136test_template (t ,"is lt" , 1137"{{ 'yes' if 2 is lt(3) }}" , 1138 json::object (), 1139"yes" 1140 ); 1141 1142test_template (t ,"is lessthan" , 1143"{{ 'yes' if 2 is lessthan(3) }}" , 1144 json::object (), 1145"yes" 1146 ); 1147 1148test_template (t ,"is ne" , 1149"{{ 'yes' if 2 is ne(3) }}" , 1150 json::object (), 1151"yes" 1152 ); 1153 1154test_template (t ,"is lower" , 1155"{{ 'yes' if 'lowercase' is lower }}" , 1156 json::object (), 1157"yes" 1158 ); 1159 1160test_template (t ,"is upper" , 1161"{{ 'yes' if 'UPPERCASE' is upper }}" , 1162 json::object (), 1163"yes" 1164 ); 1165 1166test_template (t ,"is sameas" , 1167"{{ 'yes' if x is sameas(false) }}" , 1168 {{"x" , false}}, 1169"yes" 1170 ); 1171 1172test_template (t ,"is boolean" , 1173"{{ 'yes' if x is boolean }}" , 1174 {{"x" , true}}, 1175"yes" 1176 ); 1177 1178test_template (t ,"is callable" , 1179"{{ 'yes' if ''.strip is callable }}" , 1180 json::object (), 1181"yes" 1182 ); 1183 1184test_template (t ,"is escaped" , 1185"{{ 'yes' if 'foo'|safe is escaped }}" , 1186 json::object (), 1187"yes" 1188 ); 1189 1190test_template (t ,"is filter" , 1191"{{ 'yes' if 'trim' is filter }}" , 1192 json::object (), 1193"yes" 1194 ); 1195 1196test_template (t ,"is float" , 1197"{{ 'yes' if x is float }}" , 1198 {{"x" ,1.1 }}, 1199"yes" 1200 ); 1201 1202test_template (t ,"is integer" , 1203"{{ 'yes' if x is integer }}" , 1204 {{"x" ,1 }}, 1205"yes" 1206 ); 1207 1208test_template (t ,"is sequence" , 1209"{{ 'yes' if x is sequence }}" , 1210 {{"x" , json::array ({1 ,2 ,3 })}}, 1211"yes" 1212 ); 1213 1214test_template (t ,"is test" , 1215"{{ 'yes' if 'sequence' is test }}" , 1216 json::object (), 1217"yes" 1218 ); 1219 1220test_template (t ,"is undefined" , 1221"{{ 'yes' if x is undefined }}" , 1222 json::object (), 1223"yes" 1224 ); 1225 1226test_template (t ,"is none" , 1227"{% if x is none %}yes{% endif %}" , 1228 {{"x" ,nullptr }}, 1229"yes" 1230 ); 1231 1232test_template (t ,"is string" , 1233"{% if x is string %}yes{% endif %}" , 1234 {{"x" ,"hello" }}, 1235"yes" 1236 ); 1237 1238test_template (t ,"is number" , 1239"{% if x is number %}yes{% endif %}" , 1240 {{"x" ,42 }}, 1241"yes" 1242 ); 1243 1244test_template (t ,"is iterable" , 1245"{% if x is iterable %}yes{% endif %}" , 1246 {{"x" , json::array ({1 ,2 ,3 })}}, 1247"yes" 1248 ); 1249 1250test_template (t ,"is mapping" , 1251"{% if x is mapping %}yes{% endif %}" , 1252 {{"x" , {{"a" ,1 }}}}, 1253"yes" 1254 ); 1255 1256test_template (t ,"undefined is sequence" , 1257"{{ 'yes' if x is sequence }}" , 1258 json::object (), 1259"yes" 1260 ); 1261 1262test_template (t ,"undefined is iterable" , 1263"{{ 'yes' if x is iterable }}" , 1264 json::object (), 1265"yes" 1266 ); 1267 1268test_template (t ,"is in (array, true)" , 1269"{{ 'yes' if 2 is in([1, 2, 3]) }}" , 1270 json::object (), 1271"yes" 1272 ); 1273 1274test_template (t ,"is in (array, false)" , 1275"{{ 'yes' if 5 is in([1, 2, 3]) else 'no' }}" , 1276 json::object (), 1277"no" 1278 ); 1279 1280test_template (t ,"is in (string)" , 1281"{{ 'yes' if 'bc' is in('abcde') }}" , 1282 json::object (), 1283"yes" 1284 ); 1285 1286test_template (t ,"is in (object keys)" , 1287"{{ 'yes' if 'a' is in(obj) }}" , 1288 {{"obj" , {{"a" ,1 }, {"b" ,2 }}}}, 1289"yes" 1290 ); 1291 1292test_template (t ,"reject with in test" , 1293"{{ items | reject('in', skip) | join(', ') }}" , 1294 {{"items" , json::array ({"a" ,"b" ,"c" ,"d" })}, {"skip" , json::array ({"b" ,"d" })}}, 1295"a, c" 1296 ); 1297 1298test_template (t ,"select with in test" , 1299"{{ items | select('in', keep) | join(', ') }}" , 1300 {{"items" , json::array ({"a" ,"b" ,"c" ,"d" })}, {"keep" , json::array ({"b" ,"c" })}}, 1301"b, c" 1302 ); 1303} 1304 1305static void test_string_methods (testing & t ) { 1306test_template (t ,"string.upper()" , 1307"{{ s.upper() }}" , 1308 {{"s" ,"hello" }}, 1309"HELLO" 1310 ); 1311 1312test_template (t ,"string.lower()" , 1313"{{ s.lower() }}" , 1314 {{"s" ,"HELLO" }}, 1315"hello" 1316 ); 1317 1318test_template (t ,"string.strip()" , 1319"[{{ s.strip() }}]" , 1320 {{"s" ," hello " }}, 1321"[hello]" 1322 ); 1323 1324test_template (t ,"string.lstrip()" , 1325"[{{ s.lstrip() }}]" , 1326 {{"s" ," hello" }}, 1327"[hello]" 1328 ); 1329 1330test_template (t ,"string.rstrip()" , 1331"[{{ s.rstrip() }}]" , 1332 {{"s" ,"hello " }}, 1333"[hello]" 1334 ); 1335 1336test_template (t ,"string.title()" , 1337"{{ s.title() }}" , 1338 {{"s" ,"hello world" }}, 1339"Hello World" 1340 ); 1341 1342test_template (t ,"string.capitalize()" , 1343"{{ s.capitalize() }}" , 1344 {{"s" ,"heLlo World" }}, 1345"Hello world" 1346 ); 1347 1348test_template (t ,"string.startswith() true" , 1349"{% if s.startswith('hel') %}yes{% endif %}" , 1350 {{"s" ,"hello" }}, 1351"yes" 1352 ); 1353 1354test_template (t ,"string.startswith() false" , 1355"{% if s.startswith('xyz') %}yes{% else %}no{% endif %}" , 1356 {{"s" ,"hello" }}, 1357"no" 1358 ); 1359 1360test_template (t ,"string.endswith() true" , 1361"{% if s.endswith('lo') %}yes{% endif %}" , 1362 {{"s" ,"hello" }}, 1363"yes" 1364 ); 1365 1366test_template (t ,"string.endswith() false" , 1367"{% if s.endswith('xyz') %}yes{% else %}no{% endif %}" , 1368 {{"s" ,"hello" }}, 1369"no" 1370 ); 1371 1372test_template (t ,"string.split() with sep" , 1373"{{ s.split(',')|join('-') }}" , 1374 {{"s" ,"a,b,c" }}, 1375"a-b-c" 1376 ); 1377 1378test_template (t ,"string.split() with maxsplit" , 1379"{{ s.split(',', 1)|join('-') }}" , 1380 {{"s" ,"a,b,c" }}, 1381"a-b,c" 1382 ); 1383 1384test_template (t ,"string.rsplit() with sep" , 1385"{{ s.rsplit(',')|join('-') }}" , 1386 {{"s" ,"a,b,c" }}, 1387"a-b-c" 1388 ); 1389 1390test_template (t ,"string.rsplit() with maxsplit" , 1391"{{ s.rsplit(',', 1)|join('-') }}" , 1392 {{"s" ,"a,b,c" }}, 1393"a,b-c" 1394 ); 1395 1396test_template (t ,"string.replace() basic" , 1397"{{ s.replace('world', 'jinja') }}" , 1398 {{"s" ,"hello world" }}, 1399"hello jinja" 1400 ); 1401 1402test_template (t ,"string.replace() empty" , 1403"{{ s.replace('', '.') }}" , 1404 {{"s" ,"hello world" }}, 1405".h.e.l.l.o. .w.o.r.l.d." 1406 ); 1407 1408test_template (t ,"string.replace() with count" , 1409"{{ s.replace('a', 'X', 2) }}" , 1410 {{"s" ,"banana" }}, 1411"bXnXna" 1412 ); 1413 1414test_template (t ,"string.format() auto numbering" , 1415"{{ '<{}|{}>'.format(s, 42) }}" , 1416 {{"s" ,"hello" }}, 1417"<hello|42>" 1418 ); 1419 1420test_template (t ,"string.format() manual numbering" , 1421"{{ '{1}-{0}-{1}'.format('a', 'b') }}" , 1422 json::object (), 1423"b-a-b" 1424 ); 1425 1426test_template (t ,"string.format() named fields" , 1427"{{ '{name} is {age}'.format(name='Bob', age=7) }}" , 1428 json::object (), 1429"Bob is 7" 1430 ); 1431 1432test_template (t ,"string.format() escaped braces" , 1433"{{ '{{}} {} {{x}}'.format('mid') }}" , 1434 json::object (), 1435"{} mid {x}" 1436 ); 1437 1438test_template (t ,"string.format() no fields" , 1439"{{ 'plain'.format() }}" , 1440 json::object (), 1441"plain" 1442 ); 1443 1444test_template (t ,"undefined|capitalize" , 1445"{{ arr|capitalize }}" , 1446 json::object (), 1447"" 1448 ); 1449 1450test_template (t ,"undefined|title" , 1451"{{ arr|title }}" , 1452 json::object (), 1453"" 1454 ); 1455 1456test_template (t ,"undefined|truncate" , 1457"{{ arr|truncate(9) }}" , 1458 json::object (), 1459"" 1460 ); 1461 1462test_template (t ,"undefined|upper" , 1463"{{ arr|upper }}" , 1464 json::object (), 1465"" 1466 ); 1467 1468test_template (t ,"undefined|lower" , 1469"{{ arr|lower }}" , 1470 json::object (), 1471"" 1472 ); 1473 1474test_template (t ,"undefined|replace" , 1475"{{ arr|replace('a', 'b') }}" , 1476 json::object (), 1477"" 1478 ); 1479 1480test_template (t ,"undefined|trim" , 1481"{{ arr|trim }}" , 1482 json::object (), 1483"" 1484 ); 1485 1486test_template (t ,"undefined|wordcount" , 1487"{{ arr|wordcount }}" , 1488 json::object (), 1489"0" 1490 ); 1491} 1492 1493static void test_array_methods (testing & t ) { 1494test_template (t ,"array|selectattr by attribute" , 1495"{% for item in items|selectattr('active') %}{{ item.name }} {% endfor %}" , 1496 {{"items" , json::array ({ 1497 {{"name" ,"a" }, {"active" , true}}, 1498 {{"name" ,"b" }, {"active" , false}}, 1499 {{"name" ,"c" }, {"active" , true}} 1500 })}}, 1501"a c " 1502 ); 1503 1504test_template (t ,"array|selectattr with operator" , 1505"{% for item in items|selectattr('value', 'equalto', 5) %}{{ item.name }} {% endfor %}" , 1506 {{"items" , json::array ({ 1507 {{"name" ,"a" }, {"value" ,3 }}, 1508 {{"name" ,"b" }, {"value" ,5 }}, 1509 {{"name" ,"c" }, {"value" ,5 }} 1510 })}}, 1511"b c " 1512 ); 1513 1514test_template (t ,"array|tojson" , 1515"{{ arr|tojson }}" , 1516 {{"arr" , json::array ({1 ,2 ,3 })}}, 1517"[1, 2, 3]" 1518 ); 1519 1520test_template (t ,"array|tojson with strings" , 1521"{{ arr|tojson }}" , 1522 {{"arr" , json::array ({"a" ,"b" ,"c" })}}, 1523"[\"a\", \"b\", \"c\"]" 1524 ); 1525 1526test_template (t ,"array|tojson nested" , 1527"{{ arr|tojson }}" , 1528 {{"arr" , json::array ({json::array ({1 ,2 }), json::array ({3 ,4 })})}}, 1529"[[1, 2], [3, 4]]" 1530 ); 1531 1532test_template (t ,"array|last" , 1533"{{ arr|last }}" , 1534 {{"arr" , json::array ({10 ,20 ,30 })}}, 1535"30" 1536 ); 1537 1538test_template (t ,"array|last single element" , 1539"{{ arr|last }}" , 1540 {{"arr" , json::array ({42 })}}, 1541"42" 1542 ); 1543 1544test_template (t ,"array|join with separator" , 1545"{{ arr|join(', ') }}" , 1546 {{"arr" , json::array ({"a" ,"b" ,"c" })}}, 1547"a, b, c" 1548 ); 1549 1550test_template (t ,"array|join with custom separator" , 1551"{{ arr|join(' | ') }}" , 1552 {{"arr" , json::array ({1 ,2 ,3 })}}, 1553"1 | 2 | 3" 1554 ); 1555 1556test_template (t ,"array|join default separator" , 1557"{{ arr|join }}" , 1558 {{"arr" , json::array ({"x" ,"y" ,"z" })}}, 1559"xyz" 1560 ); 1561 1562test_template (t ,"array|join attribute" , 1563"{{ arr|join(attribute='age') }}" , 1564 {{"arr" , json::array ({ 1565json ({{"name" ,"a" }, {"age" ,1 }}), 1566json ({{"name" ,"b" }, {"age" ,2 }}), 1567json ({{"name" ,"c" }, {"age" ,3 }}), 1568 })}}, 1569"123" 1570 ); 1571 1572test_template (t ,"array|join numeric attribute" , 1573"{{ arr|join(attribute=-1) }}" , 1574 {{"arr" , json::array ({json::array ({1 }), json::array ({2 }), json::array ({3 })})}}, 1575"123" 1576 ); 1577 1578test_template (t ,"array.pop() last" , 1579"{{ arr.pop() }}-{{ arr|join(',') }}" , 1580 {{"arr" , json::array ({"a" ,"b" ,"c" })}}, 1581"c-a,b" 1582 ); 1583 1584test_template (t ,"array.pop() with index" , 1585"{{ arr.pop(0) }}-{{ arr|join(',') }}" , 1586 {{"arr" , json::array ({"a" ,"b" ,"c" })}}, 1587"a-b,c" 1588 ); 1589 1590test_template (t ,"array.append()" , 1591"{% set _ = arr.append('d') %}{{ arr|join(',') }}" , 1592 {{"arr" , json::array ({"a" ,"b" ,"c" })}}, 1593"a,b,c,d" 1594 ); 1595 1596test_template (t ,"array|map with attribute" , 1597"{% for v in arr|map(attribute='age') %}{{ v }} {% endfor %}" , 1598 {{"arr" , json::array ({ 1599json ({{"name" ,"a" }, {"age" ,1 }}), 1600json ({{"name" ,"b" }, {"age" ,2 }}), 1601json ({{"name" ,"c" }, {"age" ,3 }}), 1602 })}}, 1603"1 2 3 " 1604 ); 1605 1606test_template (t ,"array|map with attribute default" , 1607"{% for v in arr|map(attribute='age', default=3) %}{{ v }} {% endfor %}" , 1608 {{"arr" , json::array ({ 1609json ({{"name" ,"a" }, {"age" ,1 }}), 1610json ({{"name" ,"b" }, {"age" ,2 }}), 1611json ({{"name" ,"c" }}), 1612 })}}, 1613"1 2 3 " 1614 ); 1615 1616test_template (t ,"array|map without attribute default" , 1617"{% for v in arr|map(attribute='age') %}{{ v }} {% endfor %}" , 1618 {{"arr" , json::array ({ 1619json ({{"name" ,"a" }, {"age" ,1 }}), 1620json ({{"name" ,"b" }, {"age" ,2 }}), 1621json ({{"name" ,"c" }}), 1622 })}}, 1623"1 2 " 1624 ); 1625 1626test_template (t ,"array|map with numeric attribute" , 1627"{% for v in arr|map(attribute=0) %}{{ v }} {% endfor %}" , 1628 {{"arr" , json::array ({ 1629 json::array ({10 ,"x" }), 1630 json::array ({20 ,"y" }), 1631 json::array ({30 ,"z" }), 1632 })}}, 1633"10 20 30 " 1634 ); 1635 1636test_template (t ,"array|map with negative attribute" , 1637"{% for v in arr|map(attribute=-1) %}{{ v }} {% endfor %}" , 1638 {{"arr" , json::array ({ 1639 json::array ({10 ,"x" }), 1640 json::array ({20 ,"y" }), 1641 json::array ({30 ,"z" }), 1642 })}}, 1643"x y z " 1644 ); 1645 1646test_template (t ,"array|map with filter" , 1647"{{ arr|map('int')|sum }}" , 1648 {{"arr" , json::array ({"1" ,"2" ,"3" })}}, 1649"6" 1650 ); 1651 1652test_template (t ,"array|min" , 1653"{{ [tool_calls_count, tool_sep_count]|min }}" , 1654 {{"tool_calls_count" ,2 }, {"tool_sep_count" ,1 }}, 1655"1" 1656 ); 1657 1658test_template (t ,"array|max" , 1659"{{ [tool_calls_count, tool_sep_count]|max }}" , 1660 {{"tool_calls_count" ,2 }, {"tool_sep_count" ,1 }}, 1661"2" 1662 ); 1663 1664test_template (t ,"array|min attribute" , 1665"{{ items|min(attribute='x') }}" , 1666 {{"items" , json::array ({ 1667json ({{"x" ,2 }}), 1668json ({{"x" ,1 }}), 1669 })}}, 1670"{'x': 1}" 1671 ); 1672 1673test_template (t ,"array|max attribute" , 1674"{{ items|max(attribute='x') }}" , 1675 {{"items" , json::array ({ 1676json ({{"x" ,2 }}), 1677json ({{"x" ,1 }}), 1678 })}}, 1679"{'x': 2}" 1680 ); 1681 1682// not used by any chat templates 1683// test_template(t, "array.insert()", 1684// "{% set _ = arr.insert(1, 'x') %}{{ arr|join(',') }}", 1685// {{"arr", json::array({"a", "b", "c"})}}, 1686// "a,x,b,c" 1687// ); 1688 1689test_template (t ,"undefined|select" , 1690"{% for item in items|select('odd') %}{{ item.name }} {% endfor %}" , 1691 json::object (), 1692"" 1693 ); 1694 1695test_template (t ,"undefined|selectattr" , 1696"{% for item in items|selectattr('active') %}{{ item.name }} {% endfor %}" , 1697 json::object (), 1698"" 1699 ); 1700 1701test_template (t ,"undefined|reject" , 1702"{% for item in items|reject('even') %}{{ item.name }} {% endfor %}" , 1703 json::object (), 1704"" 1705 ); 1706 1707test_template (t ,"undefined|rejectattr" , 1708"{% for item in items|rejectattr('active') %}{{ item.name }} {% endfor %}" , 1709 json::object (), 1710"" 1711 ); 1712 1713test_template (t ,"undefined|list" , 1714"{{ arr|list|string }}" , 1715 json::object (), 1716"[]" 1717 ); 1718 1719test_template (t ,"undefined|string" , 1720"{{ arr|string }}" , 1721 json::object (), 1722"" 1723 ); 1724 1725test_template (t ,"undefined|first" , 1726"{{ arr|first }}" , 1727 json::object (), 1728"" 1729 ); 1730 1731test_template (t ,"undefined|last" , 1732"{{ arr|last }}" , 1733 json::object (), 1734"" 1735 ); 1736 1737test_template (t ,"undefined|length" , 1738"{{ arr|length }}" , 1739 json::object (), 1740"0" 1741 ); 1742 1743test_template (t ,"undefined|join" , 1744"{{ arr|join }}" , 1745 json::object (), 1746"" 1747 ); 1748 1749test_template (t ,"undefined|sort" , 1750"{{ arr|sort|string }}" , 1751 json::object (), 1752"[]" 1753 ); 1754 1755test_template (t ,"undefined|reverse" , 1756"{{ arr|reverse|join }}" , 1757 json::object (), 1758"" 1759 ); 1760 1761test_template (t ,"undefined|map" , 1762"{% for v in arr|map(attribute='age') %}{{ v }} {% endfor %}" , 1763 json::object (), 1764"" 1765 ); 1766 1767test_template (t ,"undefined|min" , 1768"{{ arr|min }}" , 1769 json::object (), 1770"" 1771 ); 1772 1773test_template (t ,"undefined|max" , 1774"{{ arr|max }}" , 1775 json::object (), 1776"" 1777 ); 1778 1779test_template (t ,"undefined|unique" , 1780"{{ arr|unique|join }}" , 1781 json::object (), 1782"" 1783 ); 1784 1785test_template (t ,"undefined|sum" , 1786"{{ arr|sum }}" , 1787 json::object (), 1788"0" 1789 ); 1790} 1791 1792static void test_object_methods (testing & t ) { 1793test_template (t ,"object.get() existing key" , 1794"{{ obj.get('a') }}" , 1795 {{"obj" , {{"a" ,1 }, {"b" ,2 }}}}, 1796"1" 1797 ); 1798 1799test_template (t ,"object.get() missing key" , 1800"[{{ obj.get('c') is none }}]" , 1801 {{"obj" , {{"a" ,1 }}}}, 1802"[True]" 1803 ); 1804 1805test_template (t ,"object.get() missing key with default" , 1806"{{ obj.get('c', 'default') }}" , 1807 {{"obj" , {{"a" ,1 }}}}, 1808"default" 1809 ); 1810 1811test_template (t ,"object.items()" , 1812"{% for k, v in obj.items() %}{{ k }}={{ v }} {% endfor %}" , 1813 {{"obj" , {{"x" ,1 }, {"y" ,2 }}}}, 1814"x=1 y=2 " 1815 ); 1816 1817test_template (t ,"object.keys()" , 1818"{% for k in obj.keys() %}{{ k }} {% endfor %}" , 1819 {{"obj" , {{"a" ,1 }, {"b" ,2 }}}}, 1820"a b " 1821 ); 1822 1823test_template (t ,"object.values()" , 1824"{% for v in obj.values() %}{{ v }} {% endfor %}" , 1825 {{"obj" , {{"a" ,1 }, {"b" ,2 }}}}, 1826"1 2 " 1827 ); 1828 1829test_template (t ,"dictsort ascending by key" , 1830"{% for k, v in obj|dictsort %}{{ k }}={{ v }} {% endfor %}" , 1831 {{"obj" , {{"z" ,2 }, {"a" ,3 }, {"m" ,1 }}}}, 1832"a=3 m=1 z=2 " 1833 ); 1834 1835test_template (t ,"dictsort descending by key" , 1836"{% for k, v in obj|dictsort(reverse=true) %}{{ k }}={{ v }} {% endfor %}" , 1837 {{"obj" , {{"a" ,1 }, {"b" ,2 }, {"c" ,3 }}}}, 1838"c=3 b=2 a=1 " 1839 ); 1840 1841test_template (t ,"dictsort by value" , 1842"{% for k, v in obj|dictsort(by='value') %}{{ k }}={{ v }} {% endfor %}" , 1843 {{"obj" , {{"a" ,3 }, {"b" ,1 }, {"c" ,2 }}}}, 1844"b=1 c=2 a=3 " 1845 ); 1846 1847test_template (t ,"dictsort case sensitive" , 1848"{% for k, v in obj|dictsort(case_sensitive=true) %}{{ k }}={{ v }} {% endfor %}" , 1849 {{"obj" , {{"a" ,1 }, {"A" ,1 }, {"b" ,2 }, {"B" ,2 }, {"c" ,3 }}}}, 1850"A=1 B=2 a=1 b=2 c=3 " 1851 ); 1852 1853test_template (t ,"object|tojson" , 1854"{{ obj|tojson }}" , 1855 {{"obj" , {{"name" ,"test" }, {"value" ,42 }}}}, 1856"{\"name\": \"test\", \"value\": 42}" 1857 ); 1858 1859test_template (t ,"nested object|tojson" , 1860"{{ obj|tojson }}" , 1861 {{"obj" , {{"outer" , {{"inner" ,"value" }}}}}}, 1862"{\"outer\": {\"inner\": \"value\"}}" 1863 ); 1864 1865test_template (t ,"array in object|tojson" , 1866"{{ obj|tojson }}" , 1867 {{"obj" , {{"items" , json::array ({1 ,2 ,3 })}}}}, 1868"{\"items\": [1, 2, 3]}" 1869 ); 1870 1871test_template (t ,"object attribute and key access" , 1872"{{ obj.keys()|join(',') }} vs {{ obj['keys'] }} vs {{ obj.test }}" , 1873 {{"obj" , {{"keys" ,"value" }, {"test" ,"attr_value" }}}}, 1874"keys,test vs value vs attr_value" 1875 ); 1876 1877test_template (t ,"env should not have object methods" , 1878"{{ keys is undefined }} {{ obj.keys is defined }}" , 1879 {{"obj" , {{"a" ,"b" }}}}, 1880"True True" 1881 ); 1882 1883test_template (t ,"expression as object key" , 1884"{% set d = {'ab': 123} %}{{ d['a' + 'b'] == 123 }}" , 1885 json::object (), 1886"True" 1887 ); 1888 1889test_template (t ,"numeric as object key (template: Seed-OSS)" , 1890"{% set d = {1: 'a', 2: 'b'} %}{{ d[1] == 'a' and d[2] == 'b' }}" , 1891 json::object (), 1892"True" 1893 ); 1894 1895test_template (t ,"undefined|items" , 1896"{{ arr|items|join }}" , 1897 json::object (), 1898"" 1899 ); 1900} 1901 1902static void test_hasher (testing & t ) { 1903static const std::vector < std::pair < size_t ,size_t >> chunk_sizes = { 1904 {1 ,2 }, 1905 {1 ,16 }, 1906 {8 ,1 }, 1907 {1 ,1024 }, 1908 {5 ,512 }, 1909 {16 ,256 }, 1910 {45 ,122 }, 1911 {70 ,634 }, 1912 }; 1913 1914static auto random_bytes = [](size_t length )-> std::string { 1915 std::string data ; 1916data .resize (length ); 1917for (size_t i = 0 ;i < length ;++ i ) { 1918data [i ]= static_cast < char > (rand () %256 ); 1919 } 1920return data ; 1921 }; 1922 1923t .test ("state unchanged with empty input" , [](testing & t ) { 1924 jinja::hasher hasher ; 1925hasher .update ("some data" ); 1926size_t initial_state = hasher .digest (); 1927hasher .update ("" ,0 ); 1928size_t final_state = hasher .digest (); 1929t .assert_true ("Hasher state should remain unchanged" ,initial_state == final_state ); 1930 }); 1931 1932t .test ("different inputs produce different hashes" , [](testing & t ) { 1933 jinja::hasher hasher1 ; 1934hasher1 .update ("data one" ); 1935size_t hash1 = hasher1 .digest (); 1936 1937 jinja::hasher hasher2 ; 1938hasher2 .update ("data two" ); 1939size_t hash2 = hasher2 .digest (); 1940 1941t .assert_true ("Different inputs should produce different hashes" ,hash1 != hash2 ); 1942 }); 1943 1944t .test ("same inputs produce same hashes" , [](testing & t ) { 1945 jinja::hasher hasher1 ; 1946hasher1 .update ("consistent data" ); 1947size_t hash1 = hasher1 .digest (); 1948 1949 jinja::hasher hasher2 ; 1950hasher2 .update ("consistent data" ); 1951size_t hash2 = hasher2 .digest (); 1952 1953t .assert_true ("Same inputs should produce same hashes" ,hash1 == hash2 ); 1954 }); 1955 1956t .test ("property: update(a ~ b) == update(a).update(b)" , [](testing & t ) { 1957for (const auto & [size1 ,size2 ] :chunk_sizes ) { 1958 std::string data1 = random_bytes (size1 ); 1959 std::string data2 = random_bytes (size2 ); 1960 1961 jinja::hasher hasher1 ; 1962hasher1 .update (data1 ); 1963hasher1 .update (data2 ); 1964size_t hash1 = hasher1 .digest (); 1965 1966 jinja::hasher hasher2 ; 1967hasher2 .update (data1 + data2 ); 1968size_t hash2 = hasher2 .digest (); 1969 1970t .assert_true ( 1971"Hashing in multiple updates should match single update (" + std::to_string (size1 )+ ", " + std::to_string (size2 )+ ")" , 1972hash1 == hash2 ); 1973 } 1974 }); 1975 1976t .test ("property: update(a ~ b) == update(a).update(b) with more update passes" , [](testing & t ) { 1977static const std::vector < size_t > sizes = {3 ,732 ,131 ,13 ,17 ,256 ,436 ,99 ,4 }; 1978 1979 jinja::hasher hasher1 ; 1980 jinja::hasher hasher2 ; 1981 1982 std::string combined_data ; 1983for (size_t size :sizes ) { 1984 std::string data = random_bytes (size ); 1985hasher1 .update (data ); 1986combined_data += data ; 1987 } 1988 1989hasher2 .update (combined_data ); 1990size_t hash1 = hasher1 .digest (); 1991size_t hash2 = hasher2 .digest (); 1992t .assert_true ( 1993"Hashing in multiple updates should match single update with many chunks" , 1994hash1 == hash2 ); 1995 }); 1996 1997t .test ("property: non associativity of update" , [](testing & t ) { 1998for (const auto & [size1 ,size2 ] :chunk_sizes ) { 1999 std::string data1 = random_bytes (size1 ); 2000 std::string data2 = random_bytes (size2 ); 2001 2002 jinja::hasher hasher1 ; 2003hasher1 .update (data1 ); 2004hasher1 .update (data2 ); 2005size_t hash1 = hasher1 .digest (); 2006 2007 jinja::hasher hasher2 ; 2008hasher2 .update (data2 ); 2009hasher2 .update (data1 ); 2010size_t hash2 = hasher2 .digest (); 2011 2012t .assert_true ( 2013"Hashing order should matter (" + std::to_string (size1 )+ ", " + std::to_string (size2 )+ ")" , 2014hash1 != hash2 ); 2015 } 2016 }); 2017 2018t .test ("property: different lengths produce different hashes (padding block size)" , [](testing & t ) { 2019 std::string random_data = random_bytes (64 ); 2020 2021 jinja::hasher hasher1 ; 2022hasher1 .update (random_data ); 2023size_t hash1 = hasher1 .digest (); 2024 2025for (int i = 0 ;i < 16 ;++ i ) { 2026random_data .push_back ('A' );// change length 2027 jinja::hasher hasher2 ; 2028hasher2 .update (random_data ); 2029size_t hash2 = hasher2 .digest (); 2030 2031t .assert_true ("Different lengths should produce different hashes (length " + std::to_string (random_data .size ())+ ")" ,hash1 != hash2 ); 2032 2033hash1 = hash2 ; 2034 } 2035 }); 2036} 2037 2038static void test_stats (testing & t ) { 2039static auto get_stats = [](const std::string & tmpl ,const json & vars )-> jinja::value { 2040 jinja::lexer lexer ; 2041auto lexer_res = lexer .tokenize (tmpl ); 2042 2043 jinja::program prog = jinja::parse_from_tokens (lexer_res ); 2044 2045 jinja::context ctx (tmpl ); 2046 jinja::global_from_json (ctx ,json {{"val" ,vars }}, true); 2047ctx .is_get_stats = true; 2048 2049 jinja::runtime runtime (ctx ); 2050runtime .execute (prog ); 2051 2052return ctx .get_val ("val" ); 2053 }; 2054 2055t .test ("stats" , [](testing & t ) { 2056 jinja::value val = get_stats ( 2057"{{val.num}} " 2058"{{val.str}} " 2059"{{val.arr[0]}} " 2060"{{val.obj.key1}} " 2061"{{val.nested | tojson}}" , 2062// Note: the json below will be wrapped inside "val" in the context 2063json { 2064 {"num" ,1 }, 2065 {"str" ,"abc" }, 2066 {"arr" , json::array ({1 ,2 ,3 })}, 2067 {"obj" , json::object ({{"key1" ,1 }, {"key2" ,2 }, {"key3" ,3 }})}, 2068 {"nested" , json::object ({ 2069 {"inner_key1" , json::array ({1 ,2 })}, 2070 {"inner_key2" , json::object ({{"a" ,"x" }, {"b" ,"y" }})} 2071 })}, 2072 {"mixed" , json::object ({ 2073 {"used" ,1 }, 2074 {"unused" ,2 }, 2075 })}, 2076 } 2077 ); 2078 2079t .assert_true ("num is used" ,val -> at ("num" )-> stats .used ); 2080t .assert_true ("str is used" ,val -> at ("str" )-> stats .used ); 2081 2082t .assert_true ("arr is used" ,val -> at ("arr" )-> stats .used ); 2083t .assert_true ("arr[0] is used" ,val -> at ("arr" )-> at (0 )-> stats .used ); 2084t .assert_true ("arr[1] is not used" , !val -> at ("arr" )-> at (1 )-> stats .used ); 2085 2086t .assert_true ("obj is used" ,val -> at ("obj" )-> stats .used ); 2087t .assert_true ("obj.key1 is used" ,val -> at ("obj" )-> at ("key1" )-> stats .used ); 2088t .assert_true ("obj.key2 is not used" , !val -> at ("obj" )-> at ("key2" )-> stats .used ); 2089 2090t .assert_true ("inner_key1[0] is used" ,val -> at ("nested" )-> at ("inner_key1" )-> at (0 )-> stats .used ); 2091t .assert_true ("inner_key2.a is used" ,val -> at ("nested" )-> at ("inner_key2" )-> at ("a" )-> stats .used ); 2092 }); 2093} 2094 2095static void test_caps (testing & t ) { 2096static auto get_caps = [](const std::string & tmpl )-> jinja::caps { 2097 jinja::lexer lexer ; 2098auto lexer_res = lexer .tokenize (tmpl ); 2099 2100 jinja::program prog = jinja::parse_from_tokens (lexer_res ); 2101 2102return jinja::caps_get (prog ); 2103 }; 2104 2105t .test ("string content" , [](testing & t ) { 2106auto caps = get_caps ( 2107"{% for message in messages %}" 2108"{{ message['role'] + ': ' + message['content'] }}" 2109"{% endfor %}" 2110 ); 2111t .assert_true ("supports string content" ,caps .supports_string_content ); 2112t .assert_true ("does not support typed content" , !caps .supports_typed_content ); 2113 }); 2114 2115t .test ("typed content, raises on string" , [](testing & t ) { 2116// 'selectattr' is not a String filter, so it throws 2117auto caps = get_caps ( 2118"{% for message in messages %}" 2119"{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}" 2120"{{ content['text'] }}" 2121"{% endfor %}" 2122"{% endfor %}" 2123 ); 2124t .assert_true ("does not support string content" , !caps .supports_string_content ); 2125t .assert_true ("supports typed content" ,caps .supports_typed_content ); 2126 }); 2127 2128t .test ("typed content, silently drops string" , [](testing & t ) { 2129// no throw here, but content[0]['text'] is undefined for a string (MiniMax-M1 case) 2130auto caps = get_caps ( 2131"{% for message in messages %}" 2132"{{ message['content'][0]['text'] }}" 2133"{% endfor %}" 2134 ); 2135t .assert_true ("does not support string content" , !caps .supports_string_content ); 2136t .assert_true ("supports typed content" ,caps .supports_typed_content ); 2137 }); 2138} 2139 2140static void test_string_parts (testing & t ) { 2141static auto render = [](const std::string & tmpl ,const json & vars )-> jinja::string { 2142 jinja::lexer lexer ; 2143auto lexer_res = lexer .tokenize (tmpl ); 2144 2145 jinja::program ast = jinja::parse_from_tokens (lexer_res ); 2146 2147 jinja::context ctx (tmpl ); 2148 jinja::global_from_json (ctx ,vars , true); 2149 2150 jinja::runtime runtime (ctx ); 2151return runtime .gather_string_parts (runtime .execute (ast ))-> as_string (); 2152 }; 2153 2154t .test ("merge joins only the neighbours with the same type" , [](testing & t ) { 2155// "AB" comes from the input and merges, "-" comes from the template and must not 2156 jinja::string res = render ("{{ val.a }}{{ val.b }}-{{ val.c }}" , 2157json {{"val" ,json {{"a" ,"A" }, {"b" ,"B" }, {"c" ,"C" }}}}); 2158 2159if (t .assert_true ("3 parts after the merge" ,res .parts .size ()== 3 )) { 2160t .assert_true ("part 0 is the merged input" ,res .parts [0 ].val == "AB" && res .parts [0 ].is_input ); 2161t .assert_true ("part 1 is from the template" ,res .parts [1 ].val == "-" && !res .parts [1 ].is_input ); 2162t .assert_true ("part 2 is input" ,res .parts [2 ].val == "C" && res .parts [2 ].is_input ); 2163 }else { 2164t .log ("parts: " + std::to_string (res .parts .size ())+ ", rendered: " + json (res .str ()).dump ()); 2165 } 2166 }); 2167 2168} 2169 2170static void test_template_cpp (testing & t ,const std::string & name ,const std::string & tmpl ,const json & vars ,const std::string & expect ) { 2171t .test (name , [& tmpl ,& vars ,& expect ](testing & t ) { 2172 jinja::lexer lexer ; 2173auto lexer_res = lexer .tokenize (tmpl ); 2174 2175 jinja::program ast = jinja::parse_from_tokens (lexer_res ); 2176 2177 jinja::context ctx (tmpl ); 2178 jinja::global_from_json (ctx ,vars , true); 2179 2180 jinja::runtime runtime (ctx ); 2181 2182try { 2183const jinja::value results = runtime .execute (ast ); 2184auto parts = runtime .gather_string_parts (results ); 2185 2186 std::string rendered ; 2187for (const auto & part :parts -> as_string ().parts ) { 2188rendered += part .val ; 2189 } 2190 2191if (!t .assert_true ("Template render mismatch" ,expect == rendered )) { 2192t .log ("Template: " + json (tmpl ).dump ()); 2193t .log ("Expected: " + json (expect ).dump ()); 2194t .log ("Actual : " + json (rendered ).dump ()); 2195 } 2196 }catch (const jinja::not_implemented_exception & e ) { 2197t .skip (e .what ()); 2198 } 2199 }); 2200} 2201 2202// keep this in-sync with https://github.com/huggingface/transformers/blob/main/src/transformers/utils/chat_template_utils.py 2203// note: we use SandboxedEnvironment instead of ImmutableSandboxedEnvironment to allow usage of in-place array methods like append() and pop() 2204static std::string py_script = R"( 2205import jinja2 2206import jinja2.ext as jinja2_ext 2207import json 2208import sys 2209from datetime import datetime 2210from jinja2.sandbox import SandboxedEnvironment 2211 2212merged_input = json.loads(sys.stdin.buffer.read().decode("utf-8")) 2213tmpl = merged_input["tmpl"] 2214vars_json = merged_input["vars"] 2215 2216env = SandboxedEnvironment( 2217trim_blocks=True, 2218lstrip_blocks=True, 2219extensions=[jinja2_ext.loopcontrols], 2220) 2221 2222def raise_exception(message): 2223raise jinja2.exceptions.TemplateError(message) 2224 2225env.filters["tojson"] = lambda x, ensure_ascii=False, indent=None, separators=None, sort_keys=False: json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators, sort_keys=sort_keys) 2226env.globals["strftime_now"] = lambda format: datetime.now().strftime(format) 2227env.globals["raise_exception"] = raise_exception 2228 2229template = env.from_string(tmpl) 2230result = template.render(**vars_json) 2231sys.stdout.buffer.write(result.encode()) 2232)" ; 2233 2234static void test_template_py (testing & t ,const std::string & name ,const std::string & tmpl ,const json & vars ,const std::string & expect ) { 2235t .test (name , [& tmpl ,& vars ,& expect ](testing & t ) { 2236// Prepare arguments 2237json merged ; 2238merged ["tmpl" ]= json (tmpl ); 2239merged ["vars" ]= vars ; 2240 2241#ifdef _WIN32 2242const char * python_executable = "python.exe" ; 2243#else 2244const char * python_executable = "python3" ; 2245#endif 2246 2247 std::vector < std::string > args = {python_executable ,"-c" ,py_script , }; 2248 2249common_subproc subprocess ; 2250int options = subprocess_option_combined_stdout_stderr 2251 |subprocess_option_no_window 2252 |subprocess_option_inherit_environment 2253 |subprocess_option_search_user_path ; 2254 2255if (!subprocess .create (args ,options )) { 2256t .log ("Failed to create subprocess" ); 2257t .assert_true ("subprocess creation" , false); 2258return ; 2259 } 2260FILE * p_stdin = subprocess .stdin_file (); 2261 2262// Write input 2263 std::string input = merged .dump (); 2264auto written = fwrite (input .c_str (),1 ,input .size (),p_stdin ); 2265if (written != input .size ()) { 2266t .log ("Failed to write complete input to subprocess stdin" ); 2267t .assert_true ("subprocess stdin write" , false); 2268subprocess .close_stdin (); 2269subprocess .join (); 2270return ; 2271 } 2272fflush (p_stdin ); 2273subprocess .close_stdin ();// Close stdin to signal EOF to the Python process 2274 2275// Read output 2276 std::string output ; 2277char buffer [1024 ]; 2278FILE * p_stdout = subprocess .stdout_file (); 2279while (fgets (buffer ,sizeof (buffer ),p_stdout )) { 2280output += buffer ; 2281 } 2282 2283int process_return = subprocess .join (); 2284 2285if (process_return != 0 ) { 2286t .log ("Python script failed with exit code: " + std::to_string (process_return )); 2287t .log ("Output: " + output ); 2288t .assert_true ("python execution" , false); 2289return ; 2290 } 2291 2292if (!t .assert_true ("Template render mismatch" ,expect == output )) { 2293t .log ("Template: " + json (tmpl ).dump ()); 2294t .log ("Expected: " + json (expect ).dump ()); 2295t .log ("Python : " + json (output ).dump ()); 2296 } 2297 }); 2298} 2299 2300static void test_template (testing & t ,const std::string & name ,const std::string & tmpl ,const json & vars ,const std::string & expect ) { 2301if (g_python_mode ) { 2302test_template_py (t ,name ,tmpl ,vars ,expect ); 2303 }else { 2304test_template_cpp (t ,name ,tmpl ,vars ,expect ); 2305 } 2306} 2307 2308// 2309// fuzz tests to ensure no crashes occur on malformed inputs 2310// 2311 2312constexpr int JINJA_FUZZ_ITERATIONS = 100 ; 2313 2314// Helper to generate random string 2315static std::string random_string (std::mt19937 & rng ,size_t max_len ) { 2316static const char charset []= "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" ; 2317 std::uniform_int_distribution < size_t > len_dist (0 ,max_len ); 2318 std::uniform_int_distribution < size_t > char_dist (0 ,sizeof (charset )- 2 ); 2319size_t len = len_dist (rng ); 2320 std::string result ; 2321result .reserve (len ); 2322for (size_t i = 0 ;i < len ;++ i ) { 2323result += charset [char_dist (rng )]; 2324 } 2325return result ; 2326} 2327 2328// Helper to execute a fuzz test case - returns true if no crash occurred 2329static bool fuzz_test_template (const std::string & tmpl ,const json & vars ) { 2330try { 2331// printf("Fuzz testing template: %s\n", tmpl.c_str()); 2332 jinja::lexer lexer ; 2333auto lexer_res = lexer .tokenize (tmpl ); 2334 jinja::program ast = jinja::parse_from_tokens (lexer_res ); 2335 jinja::context ctx (tmpl ); 2336 jinja::global_from_json (ctx ,vars , true); 2337 jinja::runtime runtime (ctx ); 2338const jinja::value results = runtime .execute (ast ); 2339runtime .gather_string_parts (results ); 2340return true;// success 2341 }catch (const std::exception & ) { 2342return true;// exception is acceptable, not a crash 2343 }catch (...) { 2344return true;// any exception is acceptable, not a crash 2345 } 2346} 2347 2348static void test_fuzzing (testing & t ) { 2349const int num_iterations = JINJA_FUZZ_ITERATIONS ; 2350const unsigned int seed = 42 ;// fixed seed for reproducibility 2351 std::mt19937 rng (seed ); 2352 2353// Distribution helpers 2354 std::uniform_int_distribution < int > choice_dist (0 ,100 ); 2355 std::uniform_int_distribution < int > int_dist (-1000 ,1000 ); 2356 std::uniform_int_distribution < size_t > idx_dist (0 ,1000 ); 2357 2358// Template fragments for fuzzing 2359const std::vector < std::string > var_names = { 2360"x" ,"y" ,"z" ,"arr" ,"obj" ,"items" ,"foo" ,"bar" ,"undefined_var" , 2361"none" ,"true" ,"false" ,"None" ,"True" ,"False" 2362 }; 2363const std::vector < std::string > filters = { 2364"length" ,"first" ,"last" ,"reverse" ,"sort" ,"unique" ,"join" ,"upper" ,"lower" , 2365"trim" ,"default" ,"tojson" ,"string" ,"int" ,"float" ,"abs" ,"list" ,"dictsort" 2366 }; 2367const std::vector < std::string > builtins = { 2368"range" ,"len" ,"dict" ,"list" ,"join" ,"str" ,"int" ,"float" ,"namespace" 2369 }; 2370 2371t .test ("out of bound array access" , [& ](testing & t ) { 2372for (int i = 0 ;i < num_iterations ;++ i ) { 2373int idx = int_dist (rng ); 2374 std::string tmpl = "{{ arr[" + std::to_string (idx )+ "] }}" ; 2375json vars = {{"arr" , json::array ({1 ,2 ,3 })}}; 2376t .assert_true ("should not crash" ,fuzz_test_template (tmpl ,vars )); 2377 } 2378 }); 2379 2380t .test ("non-existing variables" , [& ](testing & t ) { 2381for (int i = 0 ;i < num_iterations ;++ i ) { 2382 std::string var = random_string (rng ,20 ); 2383 std::string tmpl = "{{ " + var + " }}" ; 2384json vars = json::object ();// empty context 2385t .assert_true ("should not crash" ,fuzz_test_template (tmpl ,vars )); 2386 } 2387 }); 2388 2389t .test ("non-existing nested attributes" , [& ](testing & t ) { 2390for (int i = 0 ;i < num_iterations ;++ i ) { 2391 std::string var1 = var_names [choice_dist (rng ) %var_names .size ()]; 2392 std::string var2 = random_string (rng ,10 ); 2393 std::string var3 = random_string (rng ,10 ); 2394 std::string tmpl = "{{ " + var1 + "." + var2 + "." + var3 + " }}" ; 2395json vars = {{var1 , {{"other" ,123 }}}}; 2396t .assert_true ("should not crash" ,fuzz_test_template (tmpl ,vars )); 2397 } 2398 }); 2399 2400t .test ("invalid filter arguments" , [& ](testing & t ) { 2401for (int i = 0 ;i < num_iterations ;++ i ) { 2402 std::string filter = filters [choice_dist (rng ) %filters .size ()]; 2403int val = int_dist (rng ); 2404 std::string tmpl = "{{ " + std::to_string (val )+ " | " + filter + " }}" ; 2405json vars = json::object (); 2406t .assert_true ("should not crash" ,fuzz_test_template (tmpl ,vars )); 2407 } 2408 }); 2409 2410t .test ("chained filters on various types" , [& ](testing & t ) { 2411for (int i = 0 ;i < num_iterations ;++ i ) { 2412 std::string f1 = filters [choice_dist (rng ) %filters .size ()]; 2413 std::string f2 = filters [choice_dist (rng ) %filters .size ()]; 2414 std::string var = var_names [choice_dist (rng ) %var_names .size ()]; 2415 std::string tmpl = "{{ " + var + " | " + f1 + " | " + f2 + " }}" ; 2416json vars = { 2417 {"x" ,42 }, 2418 {"y" ,"hello" }, 2419 {"arr" , json::array ({1 ,2 ,3 })}, 2420 {"obj" , {{"a" ,1 }, {"b" ,2 }}}, 2421 {"items" , json::array ({"a" ,"b" ,"c" })} 2422 }; 2423t .assert_true ("should not crash" ,fuzz_test_template (tmpl ,vars )); 2424 } 2425 }); 2426 2427t .test ("invalid builtin calls" , [& ](testing & t ) { 2428for (int i = 0 ;i < num_iterations ;++ i ) { 2429 std::string builtin = builtins [choice_dist (rng ) %builtins .size ()]; 2430 std::string arg ; 2431int arg_type = choice_dist (rng ) %4 ; 2432switch (arg_type ) { 2433case 0 :arg = "\"not a number\"" ;break ; 2434case 1 :arg = "none" ;break ; 2435case 2 :arg = std::to_string (int_dist (rng ));break ; 2436case 3 :arg = "[]" ;break ; 2437 } 2438 std::string tmpl = "{{ " + builtin + "(" + arg + ") }}" ; 2439json vars = json::object (); 2440t .assert_true ("should not crash" ,fuzz_test_template (tmpl ,vars )); 2441 } 2442 }); 2443 2444t .test ("macro edge cases" , [& ](testing & t ) { 2445// Macro with no args called with args 2446t .assert_true ("macro no args with args" ,fuzz_test_template ( 2447"{% macro foo() %}hello{% endmacro %}{{ foo(1, 2, 3) }}" , 2448 json::object () 2449 )); 2450 2451// Macro with args called with no args 2452t .assert_true ("macro with args no args" ,fuzz_test_template ( 2453"{% macro foo(a, b, c) %}{{ a }}{{ b }}{{ c }}{% endmacro %}{{ foo() }}" , 2454 json::object () 2455 )); 2456 2457// Recursive macro reference 2458t .assert_true ("recursive macro" ,fuzz_test_template ( 2459"{% macro foo(n) %}{% if n > 0 %}{{ foo(n - 1) }}{% endif %}{% endmacro %}{{ foo(5) }}" , 2460 json::object () 2461 )); 2462 2463// Nested macro definitions 2464for (int i = 0 ;i < num_iterations /10 ;++ i ) { 2465 std::string tmpl = "{% macro outer() %}{% macro inner() %}x{% endmacro %}{{ inner() }}{% endmacro %}{{ outer() }}" ; 2466t .assert_true ("nested macro" ,fuzz_test_template (tmpl , json::object ())); 2467 } 2468 }); 2469 2470t .test ("empty and none operations" , [& ](testing & t ) { 2471const std::vector < std::string > empty_tests = { 2472"{{ \"\" | first }}" , 2473"{{ \"\" | last }}" , 2474"{{ [] | first }}" , 2475"{{ [] | last }}" , 2476"{{ none.attr }}" , 2477"{{ none | length }}" , 2478"{{ none | default('fallback') }}" , 2479"{{ {} | first }}" , 2480"{{ {} | dictsort }}" , 2481 }; 2482for (const auto & tmpl :empty_tests ) { 2483t .assert_true ("empty/none: " + tmpl ,fuzz_test_template (tmpl , json::object ())); 2484 } 2485 }); 2486 2487t .test ("arithmetic edge cases" , [& ](testing & t ) { 2488const std::vector < std::string > arith_tests = { 2489"{{ 1 / 0 }}" , 2490"{{ 1 // 0 }}" , 2491"{{ 1 % 0 }}" , 2492"{{ 999999999999999999 * 999999999999999999 }}" , 2493"{{ -999999999999999999 - 999999999999999999 }}" , 2494"{{ 1.0 / 0.0 }}" , 2495"{{ 0.0 / 0.0 }}" , 2496 }; 2497for (const auto & tmpl :arith_tests ) { 2498t .assert_true ("arith: " + tmpl ,fuzz_test_template (tmpl , json::object ())); 2499 } 2500 }); 2501 2502t .test ("deeply nested structures" , [& ](testing & t ) { 2503// Deeply nested loops 2504for (int depth = 1 ;depth <=10 ;++ depth ) { 2505 std::string tmpl ; 2506for (int d = 0 ;d < depth ;++ d ) { 2507tmpl += "{% for i" + std::to_string (d )+ " in arr %}" ; 2508 } 2509tmpl += "x" ; 2510for (int d = 0 ;d < depth ;++ d ) { 2511tmpl += "{% endfor %}" ; 2512 } 2513json vars = {{"arr" , json::array ({1 ,2 })}}; 2514t .assert_true ("nested loops depth " + std::to_string (depth ),fuzz_test_template (tmpl ,vars )); 2515 } 2516 2517// Deeply nested conditionals 2518for (int depth = 1 ;depth <=10 ;++ depth ) { 2519 std::string tmpl ; 2520for (int d = 0 ;d < depth ;++ d ) { 2521tmpl += "{% if true %}" ; 2522 } 2523tmpl += "x" ; 2524for (int d = 0 ;d < depth ;++ d ) { 2525tmpl += "{% endif %}" ; 2526 } 2527t .assert_true ("nested ifs depth " + std::to_string (depth ),fuzz_test_template (tmpl , json::object ())); 2528 } 2529 }); 2530 2531t .test ("special characters in strings" , [& ](testing & t ) { 2532const std::vector < std::string > special_tests = { 2533"{{ \"}{%\" }}" , 2534"{{ \"}}{{\" }}" , 2535"{{ \"{%%}\" }}" , 2536"{{ \"\\n\\t\\r\" }}" , 2537"{{ \"'\\\"'\" }}" , 2538"{{ \"hello\\x00world\" }}" , 2539 }; 2540for (const auto & tmpl :special_tests ) { 2541t .assert_true ("special: " + tmpl ,fuzz_test_template (tmpl , json::object ())); 2542 } 2543 }); 2544 2545t .test ("random template generation" , [& ](testing & t ) { 2546const std::vector < std::string > fragments = { 2547"{{ x }}" ,"{{ y }}" ,"{{ arr }}" ,"{{ obj }}" , 2548"{% if true %}a{% endif %}" , 2549"{% if false %}b{% else %}c{% endif %}" , 2550"{% for i in arr %}{{ i }}{% endfor %}" , 2551"{{ x | length }}" ,"{{ x | first }}" ,"{{ x | default(0) }}" , 2552"{{ x + y }}" ,"{{ x - y }}" ,"{{ x * y }}" , 2553"{{ x == y }}" ,"{{ x != y }}" ,"{{ x > y }}" , 2554"{{ range(3) }}" ,"{{ \"hello\" | upper }}" , 2555"text" ," " ,"\n" , 2556 }; 2557 2558for (int i = 0 ;i < num_iterations ;++ i ) { 2559 std::string tmpl ; 2560int num_frags = choice_dist (rng ) %10 + 1 ; 2561for (int f = 0 ;f < num_frags ;++ f ) { 2562tmpl += fragments [choice_dist (rng ) %fragments .size ()]; 2563 } 2564json vars = { 2565 {"x" ,int_dist (rng )}, 2566 {"y" ,int_dist (rng )}, 2567 {"arr" , json::array ({1 ,2 ,3 })}, 2568 {"obj" , {{"a" ,1 }, {"b" ,2 }}} 2569 }; 2570t .assert_true ("random template #" + std::to_string (i ),fuzz_test_template (tmpl ,vars )); 2571 } 2572 }); 2573 2574t .test ("malformed templates (should error, not crash)" , [& ](testing & t ) { 2575const std::vector < std::string > malformed = { 2576"" , 2577"{{ x" , 2578"{% if %}" , 2579"{% for %}" , 2580"{% for x in %}" , 2581"{% endfor %}" , 2582"{% endif %}" , 2583"{{ | filter }}" , 2584"{% if x %}" ,// unclosed 2585"{% for i in x %}" ,// unclosed 2586"{{ x | }}" , 2587"{% macro %}{% endmacro %}" , 2588"{{{{" , 2589"}}}}" , 2590"{%%}" , 2591"{% set %}" , 2592"{% set x %}" , 2593 }; 2594for (const auto & tmpl :malformed ) { 2595t .assert_true ("malformed: " + tmpl ,fuzz_test_template (tmpl , json::object ())); 2596 } 2597 std::string tmpl = "{% for message in messages %}{{ message.role | string }} : {{ message.content if ('content' in message and message.content is not none) }}{% endfor %" ; 2598while (tmpl .length ()> 0 ) { 2599t .assert_true ("malformed: " + tmpl ,fuzz_test_template (tmpl , json::object ())); 2600tmpl .pop_back (); 2601 } 2602 }); 2603 2604t .test ("type coercion edge cases" , [& ](testing & t ) { 2605for (int i = 0 ;i < num_iterations ;++ i ) { 2606int op_choice = choice_dist (rng ) %6 ; 2607 std::string op ; 2608switch (op_choice ) { 2609case 0 :op = "+" ;break ; 2610case 1 :op = "-" ;break ; 2611case 2 :op = "*" ;break ; 2612case 3 :op = "/" ;break ; 2613case 4 :op = "==" ;break ; 2614case 5 :op = "~" ;break ;// string concat 2615 } 2616 2617 std::string left_var = var_names [choice_dist (rng ) %var_names .size ()]; 2618 std::string right_var = var_names [choice_dist (rng ) %var_names .size ()]; 2619 std::string tmpl = "{{ " + left_var + " " + op + " " + right_var + " }}" ; 2620 2621json vars = { 2622 {"x" ,42 }, 2623 {"y" ,"hello" }, 2624 {"z" ,3.14 }, 2625 {"arr" , json::array ({1 ,2 ,3 })}, 2626 {"obj" , {{"a" ,1 }}}, 2627 {"items" , json::array ()}, 2628 {"foo" ,nullptr }, 2629 {"bar" , true} 2630 }; 2631t .assert_true ("type coercion: " + tmpl ,fuzz_test_template (tmpl ,vars )); 2632 } 2633 }); 2634 2635t .test ("fuzz builtin functions" , [& ](testing & t ) { 2636// pair of (type_name, builtin_name) 2637 std::vector < std::pair < std::string , std::string >> builtins ; 2638auto add_fns = [& ](std::string type_name ,const jinja::func_builtins & added ) { 2639for (const auto & it :added ) { 2640builtins .push_back ({type_name ,it .first }); 2641 } 2642 }; 2643add_fns ("global" , jinja::global_builtins ()); 2644add_fns ("int" , jinja::value_int_t (0 ).get_builtins ()); 2645add_fns ("float" , jinja::value_float_t (0.0f ).get_builtins ()); 2646add_fns ("string" , jinja::value_string_t ().get_builtins ()); 2647add_fns ("array" , jinja::value_array_t ().get_builtins ()); 2648add_fns ("object" , jinja::value_object_t ().get_builtins ()); 2649 2650const int max_args = 5 ; 2651const std::vector < std::string > kwarg_names = { 2652"base" ,"attribute" ,"default" ,"reverse" ,"case_sensitive" ,"by" ,"safe" ,"chars" ,"separators" ,"sort_keys" ,"indent" ,"ensure_ascii" , 2653 }; 2654 2655// Generate random argument values 2656auto gen_random_arg = [& ]()-> std::string { 2657int type = choice_dist (rng ) %8 ; 2658switch (type ) { 2659case 0 :return std::to_string (int_dist (rng ));// int 2660case 1 :return std::to_string (int_dist (rng ))+ ".5" ;// float 2661case 2 :return "\"" + random_string (rng ,10 )+ "\"" ;// string 2662case 3 :return "true" ;// bool true 2663case 4 :return "false" ;// bool false 2664case 5 :return "none" ;// none 2665case 6 :return "[1, 2, 3]" ;// array 2666case 7 :return "{\"a\": 1}" ;// object 2667default :return "0" ; 2668 } 2669 }; 2670 2671for (int i = 0 ;i < num_iterations ;++ i ) { 2672// Pick a random builtin 2673auto & [type_name ,fn_name ]= builtins [choice_dist (rng ) %builtins .size ()]; 2674 2675// Generate random number of args 2676int num_args = choice_dist (rng ) % (max_args + 1 ); 2677 std::string args_str ; 2678for (int a = 0 ;a < num_args ;++ a ) { 2679if (a > 0 )args_str += ", " ; 2680// Sometimes use keyword args 2681if (choice_dist (rng ) %3 == 0 && !kwarg_names .empty ()) { 2682 std::string kwarg = kwarg_names [choice_dist (rng ) %kwarg_names .size ()]; 2683args_str += kwarg + "=" + gen_random_arg (); 2684 }else { 2685args_str += gen_random_arg (); 2686 } 2687 } 2688 2689 std::string tmpl ; 2690if (type_name == "global" ) { 2691// Global function call 2692tmpl = "{{ " + fn_name + "(" + args_str + ") }}" ; 2693 }else { 2694// Method call on a value 2695 std::string base_val ; 2696if (type_name == "int" ) { 2697base_val = std::to_string (int_dist (rng )); 2698 }else if (type_name == "float" ) { 2699base_val = std::to_string (int_dist (rng ))+ ".5" ; 2700 }else if (type_name == "string" ) { 2701base_val = "\"test_string\"" ; 2702 }else if (type_name == "array" ) { 2703base_val = "[1, 2, 3, \"a\", \"b\"]" ; 2704 }else if (type_name == "object" ) { 2705base_val = "{\"x\": 1, \"y\": 2}" ; 2706 }else { 2707base_val = "x" ; 2708 } 2709tmpl = "{{ " + base_val + "." + fn_name + "(" + args_str + ") }}" ; 2710 } 2711 2712json vars = { 2713 {"x" ,42 }, 2714 {"y" ,"hello" }, 2715 {"arr" , json::array ({1 ,2 ,3 })}, 2716 {"obj" , {{"a" ,1 }, {"b" ,2 }}} 2717 }; 2718 2719t .assert_true ("builtin " + type_name + "." + fn_name + " #" + std::to_string (i ),fuzz_test_template (tmpl ,vars )); 2720 } 2721 }); 2722 2723t .test ("tojson ensure_ascii=true with invalid utf-8" , [& ](testing & t ) { 2724t .assert_true ("invalid utf-8 does not crash" , 2725fuzz_test_template ( 2726"{{ data|tojson(ensure_ascii=true) }}" , 2727 {{"data" , std::string ("hello\xfe\xffworld" )}} 2728 )); 2729 }); 2730}