zclsyntax: scan single-character tokens that represent themselves
For convenience we use the rune values of these tokens as their token enum values, so we can handle them all via a single rule.
This commit is contained in:
Martin Atkins
parent
53944335f1
commit
5898b36695
@ -12,8 +12,8 @@ var _zcltok_actions []byte = []byte{
|
||||
0, 1, 0, 1, 1, 1, 2, 1, 5,
|
||||
1, 6, 1, 7, 1, 8, 1, 9,
|
||||
1, 10, 1, 11, 1, 12, 1, 13,
|
||||
1, 14, 1, 15, 2, 2, 3, 2,
|
||||
2, 4,
|
||||
1, 14, 1, 15, 1, 16, 2, 2,
|
||||
3, 2, 2, 4,
|
||||
}
|
||||
|
||||
var _zcltok_key_offsets []int16 = []int16{
|
||||
@ -81,12 +81,12 @@ var _zcltok_key_offsets []int16 = []int16{
|
||||
3163, 3181, 3182, 3192, 3193, 3202, 3210, 3212,
|
||||
3215, 3217, 3219, 3221, 3226, 3239, 3243, 3258,
|
||||
3287, 3298, 3300, 3304, 3308, 3313, 3317, 3319,
|
||||
3326, 3330, 3338, 3342, 3398, 3399, 3400, 3405,
|
||||
3448, 3450, 3455, 3459, 3459, 3461, 3463, 3474,
|
||||
3484, 3492, 3493, 3495, 3496, 3500, 3504, 3514,
|
||||
3518, 3525, 3536, 3543, 3547, 3553, 3564, 3596,
|
||||
3645, 3660, 3675, 3680, 3682, 3687, 3719, 3727,
|
||||
3729, 3751, 3773,
|
||||
3326, 3330, 3338, 3342, 3408, 3409, 3410, 3415,
|
||||
3458, 3460, 3465, 3469, 3469, 3471, 3473, 3484,
|
||||
3494, 3502, 3503, 3505, 3506, 3510, 3514, 3524,
|
||||
3528, 3535, 3546, 3553, 3557, 3563, 3574, 3606,
|
||||
3655, 3670, 3685, 3690, 3692, 3697, 3729, 3737,
|
||||
3739, 3761, 3783,
|
||||
}
|
||||
|
||||
var _zcltok_trans_keys []byte = []byte{
|
||||
@ -507,61 +507,63 @@ var _zcltok_trans_keys []byte = []byte{
|
||||
191, 181, 191, 192, 255, 158, 159, 186,
|
||||
128, 185, 187, 191, 192, 255, 162, 191,
|
||||
192, 255, 160, 168, 128, 159, 161, 167,
|
||||
169, 191, 158, 191, 192, 255, 9, 32,
|
||||
169, 191, 158, 191, 192, 255, 9, 10,
|
||||
32, 33, 38, 44, 64, 92, 95, 127,
|
||||
194, 195, 198, 199, 203, 204, 205, 206,
|
||||
207, 210, 212, 213, 214, 215, 216, 217,
|
||||
219, 220, 221, 222, 223, 224, 225, 226,
|
||||
227, 228, 233, 234, 237, 238, 239, 240,
|
||||
0, 47, 48, 57, 58, 64, 65, 90,
|
||||
91, 96, 97, 122, 123, 127, 192, 193,
|
||||
196, 218, 229, 236, 241, 247, 9, 32,
|
||||
46, 69, 101, 48, 57, 95, 194, 195,
|
||||
198, 199, 203, 204, 205, 206, 207, 210,
|
||||
212, 213, 214, 215, 216, 217, 219, 220,
|
||||
221, 222, 223, 224, 225, 226, 227, 228,
|
||||
233, 234, 237, 239, 240, 243, 48, 57,
|
||||
65, 90, 97, 122, 196, 218, 229, 236,
|
||||
128, 191, 170, 181, 186, 128, 191, 151,
|
||||
183, 128, 255, 192, 255, 0, 127, 173,
|
||||
130, 133, 146, 159, 165, 171, 175, 191,
|
||||
192, 255, 181, 190, 128, 175, 176, 183,
|
||||
184, 185, 186, 191, 134, 139, 141, 162,
|
||||
128, 135, 136, 255, 182, 130, 137, 176,
|
||||
151, 152, 154, 160, 136, 191, 192, 255,
|
||||
128, 143, 144, 170, 171, 175, 176, 178,
|
||||
179, 191, 128, 159, 160, 191, 176, 128,
|
||||
138, 139, 173, 174, 255, 148, 150, 164,
|
||||
167, 173, 176, 185, 189, 190, 192, 255,
|
||||
144, 128, 145, 146, 175, 176, 191, 128,
|
||||
140, 141, 255, 166, 176, 178, 191, 192,
|
||||
255, 186, 128, 137, 138, 170, 171, 179,
|
||||
180, 181, 182, 191, 160, 161, 162, 164,
|
||||
165, 166, 167, 168, 169, 170, 171, 172,
|
||||
173, 174, 175, 176, 177, 178, 179, 180,
|
||||
181, 182, 183, 184, 185, 186, 187, 188,
|
||||
189, 190, 128, 191, 128, 129, 130, 131,
|
||||
137, 138, 139, 140, 141, 142, 143, 144,
|
||||
153, 154, 155, 156, 157, 158, 159, 160,
|
||||
161, 162, 163, 164, 165, 166, 167, 168,
|
||||
169, 170, 171, 172, 173, 174, 175, 176,
|
||||
177, 178, 179, 180, 182, 183, 184, 188,
|
||||
189, 190, 191, 132, 187, 129, 130, 132,
|
||||
133, 134, 176, 177, 178, 179, 180, 181,
|
||||
182, 183, 128, 191, 128, 129, 130, 131,
|
||||
132, 133, 134, 135, 144, 136, 143, 145,
|
||||
191, 192, 255, 182, 183, 184, 128, 191,
|
||||
128, 191, 191, 128, 190, 192, 255, 128,
|
||||
146, 147, 148, 152, 153, 154, 155, 156,
|
||||
158, 159, 160, 161, 162, 163, 164, 165,
|
||||
166, 167, 168, 169, 170, 171, 172, 173,
|
||||
174, 175, 176, 129, 191, 192, 255, 158,
|
||||
159, 128, 157, 160, 191, 192, 255, 128,
|
||||
191, 164, 169, 171, 172, 173, 174, 175,
|
||||
180, 181, 182, 183, 184, 185, 187, 188,
|
||||
189, 190, 191, 128, 163, 165, 186, 144,
|
||||
145, 146, 147, 148, 150, 151, 152, 155,
|
||||
157, 158, 160, 170, 171, 172, 175, 128,
|
||||
159, 161, 169, 173, 191, 128, 191,
|
||||
0, 39, 40, 47, 48, 57, 58, 63,
|
||||
65, 90, 91, 96, 97, 122, 123, 126,
|
||||
192, 193, 196, 218, 229, 236, 241, 247,
|
||||
9, 32, 46, 69, 101, 48, 57, 95,
|
||||
194, 195, 198, 199, 203, 204, 205, 206,
|
||||
207, 210, 212, 213, 214, 215, 216, 217,
|
||||
219, 220, 221, 222, 223, 224, 225, 226,
|
||||
227, 228, 233, 234, 237, 239, 240, 243,
|
||||
48, 57, 65, 90, 97, 122, 196, 218,
|
||||
229, 236, 128, 191, 170, 181, 186, 128,
|
||||
191, 151, 183, 128, 255, 192, 255, 0,
|
||||
127, 173, 130, 133, 146, 159, 165, 171,
|
||||
175, 191, 192, 255, 181, 190, 128, 175,
|
||||
176, 183, 184, 185, 186, 191, 134, 139,
|
||||
141, 162, 128, 135, 136, 255, 182, 130,
|
||||
137, 176, 151, 152, 154, 160, 136, 191,
|
||||
192, 255, 128, 143, 144, 170, 171, 175,
|
||||
176, 178, 179, 191, 128, 159, 160, 191,
|
||||
176, 128, 138, 139, 173, 174, 255, 148,
|
||||
150, 164, 167, 173, 176, 185, 189, 190,
|
||||
192, 255, 144, 128, 145, 146, 175, 176,
|
||||
191, 128, 140, 141, 255, 166, 176, 178,
|
||||
191, 192, 255, 186, 128, 137, 138, 170,
|
||||
171, 179, 180, 181, 182, 191, 160, 161,
|
||||
162, 164, 165, 166, 167, 168, 169, 170,
|
||||
171, 172, 173, 174, 175, 176, 177, 178,
|
||||
179, 180, 181, 182, 183, 184, 185, 186,
|
||||
187, 188, 189, 190, 128, 191, 128, 129,
|
||||
130, 131, 137, 138, 139, 140, 141, 142,
|
||||
143, 144, 153, 154, 155, 156, 157, 158,
|
||||
159, 160, 161, 162, 163, 164, 165, 166,
|
||||
167, 168, 169, 170, 171, 172, 173, 174,
|
||||
175, 176, 177, 178, 179, 180, 182, 183,
|
||||
184, 188, 189, 190, 191, 132, 187, 129,
|
||||
130, 132, 133, 134, 176, 177, 178, 179,
|
||||
180, 181, 182, 183, 128, 191, 128, 129,
|
||||
130, 131, 132, 133, 134, 135, 144, 136,
|
||||
143, 145, 191, 192, 255, 182, 183, 184,
|
||||
128, 191, 128, 191, 191, 128, 190, 192,
|
||||
255, 128, 146, 147, 148, 152, 153, 154,
|
||||
155, 156, 158, 159, 160, 161, 162, 163,
|
||||
164, 165, 166, 167, 168, 169, 170, 171,
|
||||
172, 173, 174, 175, 176, 129, 191, 192,
|
||||
255, 158, 159, 128, 157, 160, 191, 192,
|
||||
255, 128, 191, 164, 169, 171, 172, 173,
|
||||
174, 175, 180, 181, 182, 183, 184, 185,
|
||||
187, 188, 189, 190, 191, 128, 163, 165,
|
||||
186, 144, 145, 146, 147, 148, 150, 151,
|
||||
152, 155, 157, 158, 160, 170, 171, 172,
|
||||
175, 128, 159, 161, 169, 173, 191, 128,
|
||||
191,
|
||||
}
|
||||
|
||||
var _zcltok_single_lengths []byte = []byte{
|
||||
@ -629,7 +631,7 @@ var _zcltok_single_lengths []byte = []byte{
|
||||
12, 1, 4, 1, 5, 2, 0, 3,
|
||||
2, 2, 2, 1, 7, 0, 7, 17,
|
||||
3, 0, 2, 0, 3, 0, 0, 1,
|
||||
0, 2, 0, 34, 1, 1, 3, 33,
|
||||
0, 2, 0, 42, 1, 1, 3, 33,
|
||||
0, 3, 2, 0, 0, 0, 1, 2,
|
||||
4, 1, 0, 1, 0, 0, 0, 0,
|
||||
1, 1, 1, 0, 0, 1, 30, 47,
|
||||
@ -702,7 +704,7 @@ var _zcltok_range_lengths []byte = []byte{
|
||||
3, 0, 3, 0, 2, 3, 1, 0,
|
||||
0, 0, 0, 2, 3, 2, 4, 6,
|
||||
4, 1, 1, 2, 1, 2, 1, 3,
|
||||
2, 3, 2, 11, 0, 0, 1, 5,
|
||||
2, 3, 2, 12, 0, 0, 1, 5,
|
||||
1, 1, 1, 0, 1, 1, 5, 4,
|
||||
2, 0, 1, 0, 2, 2, 5, 2,
|
||||
3, 5, 3, 2, 3, 5, 1, 1,
|
||||
@ -775,12 +777,12 @@ var _zcltok_index_offsets []int16 = []int16{
|
||||
2421, 2437, 2439, 2447, 2449, 2457, 2463, 2465,
|
||||
2469, 2472, 2475, 2478, 2482, 2493, 2496, 2508,
|
||||
2532, 2540, 2542, 2546, 2549, 2554, 2557, 2559,
|
||||
2564, 2567, 2573, 2576, 2622, 2624, 2626, 2631,
|
||||
2670, 2672, 2677, 2681, 2682, 2684, 2686, 2693,
|
||||
2700, 2707, 2709, 2711, 2713, 2716, 2719, 2725,
|
||||
2728, 2733, 2740, 2745, 2748, 2752, 2759, 2791,
|
||||
2840, 2855, 2868, 2873, 2875, 2879, 2910, 2916,
|
||||
2918, 2939, 2959,
|
||||
2564, 2567, 2573, 2576, 2631, 2633, 2635, 2640,
|
||||
2679, 2681, 2686, 2690, 2691, 2693, 2695, 2702,
|
||||
2709, 2716, 2718, 2720, 2722, 2725, 2728, 2734,
|
||||
2737, 2742, 2749, 2754, 2757, 2761, 2768, 2800,
|
||||
2849, 2864, 2877, 2882, 2884, 2888, 2919, 2925,
|
||||
2927, 2948, 2968,
|
||||
}
|
||||
|
||||
var _zcltok_indicies []int16 = []int16{
|
||||
@ -1106,55 +1108,56 @@ var _zcltok_indicies []int16 = []int16{
|
||||
268, 10, 255, 254, 4, 255, 4, 349,
|
||||
268, 256, 254, 10, 255, 254, 4, 259,
|
||||
350, 256, 268, 256, 254, 255, 254, 4,
|
||||
351, 352, 355, 356, 358, 359, 360, 354,
|
||||
361, 362, 363, 364, 365, 366, 367, 368,
|
||||
369, 370, 371, 372, 373, 374, 375, 376,
|
||||
377, 378, 379, 380, 382, 383, 384, 385,
|
||||
386, 387, 255, 2, 255, 4, 255, 4,
|
||||
255, 354, 357, 381, 388, 353, 351, 389,
|
||||
352, 390, 2, 392, 392, 2, 391, 4,
|
||||
394, 395, 74, 55, 396, 55, 397, 398,
|
||||
351, 352, 353, 352, 352, 255, 255, 255,
|
||||
255, 255, 356, 357, 359, 360, 361, 355,
|
||||
362, 363, 364, 365, 366, 367, 368, 369,
|
||||
370, 371, 372, 373, 374, 375, 376, 377,
|
||||
378, 379, 380, 381, 383, 384, 385, 386,
|
||||
387, 388, 255, 352, 2, 352, 4, 352,
|
||||
4, 352, 355, 358, 382, 389, 354, 351,
|
||||
390, 353, 391, 2, 393, 393, 2, 392,
|
||||
4, 395, 396, 74, 55, 397, 55, 398,
|
||||
399, 400, 401, 402, 403, 404, 405, 406,
|
||||
407, 129, 408, 105, 409, 410, 411, 412,
|
||||
407, 408, 129, 409, 105, 410, 411, 412,
|
||||
413, 414, 415, 416, 417, 418, 419, 420,
|
||||
4, 4, 4, 10, 150, 393, 255, 421,
|
||||
4, 4, 4, 255, 421, 255, 255, 4,
|
||||
421, 4, 421, 4, 421, 4, 255, 255,
|
||||
255, 255, 255, 421, 4, 255, 255, 255,
|
||||
4, 255, 4, 421, 4, 255, 255, 255,
|
||||
255, 4, 421, 255, 4, 255, 4, 255,
|
||||
4, 255, 255, 4, 255, 421, 4, 255,
|
||||
4, 255, 4, 255, 421, 255, 4, 421,
|
||||
255, 4, 255, 4, 421, 255, 255, 255,
|
||||
255, 255, 421, 4, 4, 255, 4, 255,
|
||||
421, 255, 4, 421, 255, 255, 421, 4,
|
||||
4, 255, 4, 255, 4, 255, 421, 422,
|
||||
421, 4, 4, 4, 10, 150, 394, 255,
|
||||
422, 4, 4, 4, 255, 422, 255, 255,
|
||||
4, 422, 4, 422, 4, 422, 4, 255,
|
||||
255, 255, 255, 255, 422, 4, 255, 255,
|
||||
255, 4, 255, 4, 422, 4, 255, 255,
|
||||
255, 255, 4, 422, 255, 4, 255, 4,
|
||||
255, 4, 255, 255, 4, 255, 422, 4,
|
||||
255, 4, 255, 4, 255, 422, 255, 4,
|
||||
422, 255, 4, 255, 4, 422, 255, 255,
|
||||
255, 255, 255, 422, 4, 4, 255, 4,
|
||||
255, 422, 255, 4, 422, 255, 255, 422,
|
||||
4, 4, 255, 4, 255, 4, 255, 422,
|
||||
423, 424, 425, 426, 427, 428, 429, 430,
|
||||
431, 432, 300, 433, 434, 435, 436, 437,
|
||||
438, 439, 440, 441, 442, 443, 444, 443,
|
||||
445, 446, 447, 448, 449, 256, 421, 450,
|
||||
431, 432, 433, 300, 434, 435, 436, 437,
|
||||
438, 439, 440, 441, 442, 443, 444, 445,
|
||||
444, 446, 447, 448, 449, 450, 256, 422,
|
||||
451, 452, 453, 454, 455, 456, 457, 458,
|
||||
459, 460, 461, 462, 463, 464, 465, 466,
|
||||
467, 468, 310, 469, 470, 471, 277, 472,
|
||||
473, 474, 475, 476, 477, 256, 478, 479,
|
||||
480, 481, 482, 483, 484, 485, 259, 486,
|
||||
256, 259, 487, 488, 489, 490, 268, 421,
|
||||
491, 492, 493, 494, 288, 495, 496, 268,
|
||||
497, 498, 499, 500, 501, 256, 421, 502,
|
||||
461, 503, 504, 505, 268, 506, 507, 259,
|
||||
256, 268, 10, 421, 471, 256, 259, 268,
|
||||
10, 268, 10, 508, 268, 421, 10, 259,
|
||||
509, 510, 259, 511, 512, 266, 513, 514,
|
||||
515, 516, 517, 467, 518, 519, 520, 521,
|
||||
467, 468, 469, 310, 470, 471, 472, 277,
|
||||
473, 474, 475, 476, 477, 478, 256, 479,
|
||||
480, 481, 482, 483, 484, 485, 486, 259,
|
||||
487, 256, 259, 488, 489, 490, 491, 268,
|
||||
422, 492, 493, 494, 495, 288, 496, 497,
|
||||
268, 498, 499, 500, 501, 502, 256, 422,
|
||||
503, 462, 504, 505, 506, 268, 507, 508,
|
||||
259, 256, 268, 10, 422, 472, 256, 259,
|
||||
268, 10, 268, 10, 509, 268, 422, 10,
|
||||
259, 510, 511, 259, 512, 513, 266, 514,
|
||||
515, 516, 517, 518, 468, 519, 520, 521,
|
||||
522, 523, 524, 525, 526, 527, 528, 529,
|
||||
486, 530, 259, 268, 10, 421, 531, 532,
|
||||
268, 256, 421, 10, 256, 421, 259, 533,
|
||||
316, 534, 535, 536, 537, 538, 539, 540,
|
||||
541, 256, 542, 543, 544, 545, 546, 547,
|
||||
256, 268, 421, 549, 550, 551, 552, 553,
|
||||
554, 555, 556, 557, 558, 559, 555, 561,
|
||||
562, 563, 564, 548, 560, 548, 421, 548,
|
||||
421,
|
||||
530, 487, 531, 259, 268, 10, 422, 532,
|
||||
533, 268, 256, 422, 10, 256, 422, 259,
|
||||
534, 316, 535, 536, 537, 538, 539, 540,
|
||||
541, 542, 256, 543, 544, 545, 546, 547,
|
||||
548, 256, 268, 422, 550, 551, 552, 553,
|
||||
554, 555, 556, 557, 558, 559, 560, 556,
|
||||
562, 563, 564, 565, 549, 561, 549, 422,
|
||||
549, 422,
|
||||
}
|
||||
|
||||
var _zcltok_trans_targs []int16 = []int16{
|
||||
@ -1202,37 +1205,37 @@ var _zcltok_trans_targs []int16 = []int16{
|
||||
485, 486, 487, 489, 490, 491, 492, 493,
|
||||
494, 495, 496, 497, 498, 499, 501, 502,
|
||||
503, 504, 507, 509, 510, 512, 514, 516,
|
||||
517, 515, 520, 521, 522, 523, 524, 525,
|
||||
526, 527, 528, 529, 530, 531, 532, 533,
|
||||
534, 535, 536, 537, 538, 539, 540, 541,
|
||||
542, 543, 544, 545, 546, 547, 548, 549,
|
||||
550, 551, 552, 553, 554, 515, 515, 515,
|
||||
0, 515, 2, 3, 7, 8, 9, 10,
|
||||
11, 12, 13, 14, 15, 16, 17, 18,
|
||||
20, 22, 23, 55, 96, 111, 118, 120,
|
||||
122, 142, 145, 161, 274, 515, 278, 279,
|
||||
280, 281, 282, 283, 284, 285, 286, 287,
|
||||
288, 290, 291, 292, 293, 294, 295, 296,
|
||||
297, 298, 299, 300, 301, 302, 303, 304,
|
||||
305, 306, 307, 308, 309, 310, 312, 313,
|
||||
314, 315, 316, 317, 318, 319, 320, 321,
|
||||
322, 323, 324, 325, 326, 328, 329, 330,
|
||||
332, 333, 334, 335, 336, 337, 338, 339,
|
||||
340, 341, 342, 343, 344, 345, 347, 348,
|
||||
349, 350, 351, 352, 353, 354, 355, 357,
|
||||
358, 359, 360, 361, 362, 363, 364, 365,
|
||||
366, 367, 368, 369, 370, 371, 372, 373,
|
||||
374, 376, 377, 378, 379, 380, 381, 382,
|
||||
383, 384, 385, 386, 387, 388, 389, 390,
|
||||
391, 392, 393, 394, 395, 396, 398, 399,
|
||||
400, 401, 402, 403, 404, 405, 406, 407,
|
||||
408, 409, 410, 411, 412, 413, 442, 467,
|
||||
470, 471, 473, 480, 481, 484, 488, 500,
|
||||
505, 506, 508, 511, 513,
|
||||
515, 517, 515, 520, 521, 522, 523, 524,
|
||||
525, 526, 527, 528, 529, 530, 531, 532,
|
||||
533, 534, 535, 536, 537, 538, 539, 540,
|
||||
541, 542, 543, 544, 545, 546, 547, 548,
|
||||
549, 550, 551, 552, 553, 554, 515, 515,
|
||||
515, 0, 515, 2, 3, 7, 8, 9,
|
||||
10, 11, 12, 13, 14, 15, 16, 17,
|
||||
18, 20, 22, 23, 55, 96, 111, 118,
|
||||
120, 122, 142, 145, 161, 274, 515, 278,
|
||||
279, 280, 281, 282, 283, 284, 285, 286,
|
||||
287, 288, 290, 291, 292, 293, 294, 295,
|
||||
296, 297, 298, 299, 300, 301, 302, 303,
|
||||
304, 305, 306, 307, 308, 309, 310, 312,
|
||||
313, 314, 315, 316, 317, 318, 319, 320,
|
||||
321, 322, 323, 324, 325, 326, 328, 329,
|
||||
330, 332, 333, 334, 335, 336, 337, 338,
|
||||
339, 340, 341, 342, 343, 344, 345, 347,
|
||||
348, 349, 350, 351, 352, 353, 354, 355,
|
||||
357, 358, 359, 360, 361, 362, 363, 364,
|
||||
365, 366, 367, 368, 369, 370, 371, 372,
|
||||
373, 374, 376, 377, 378, 379, 380, 381,
|
||||
382, 383, 384, 385, 386, 387, 388, 389,
|
||||
390, 391, 392, 393, 394, 395, 396, 398,
|
||||
399, 400, 401, 402, 403, 404, 405, 406,
|
||||
407, 408, 409, 410, 411, 412, 413, 442,
|
||||
467, 470, 471, 473, 480, 481, 484, 488,
|
||||
500, 505, 506, 508, 511, 513,
|
||||
}
|
||||
|
||||
var _zcltok_trans_actions []byte = []byte{
|
||||
21, 0, 5, 23, 29, 27, 0, 0,
|
||||
23, 0, 5, 25, 31, 29, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
@ -1263,7 +1266,7 @@ var _zcltok_trans_actions []byte = []byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 25, 7,
|
||||
0, 0, 0, 0, 0, 0, 27, 9,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
@ -1276,15 +1279,15 @@ var _zcltok_trans_actions []byte = []byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 9, 0, 0, 0, 0, 0, 0,
|
||||
7, 0, 11, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
5, 5, 5, 32, 32, 32, 32, 32,
|
||||
32, 5, 5, 32, 5, 17, 11, 13,
|
||||
0, 15, 0, 0, 0, 0, 0, 0,
|
||||
0, 5, 5, 5, 34, 34, 34, 34,
|
||||
34, 34, 5, 5, 34, 5, 19, 13,
|
||||
15, 0, 17, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 19, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 21, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
@ -1302,7 +1305,7 @@ var _zcltok_trans_actions []byte = []byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0,
|
||||
}
|
||||
|
||||
var _zcltok_to_state_actions []byte = []byte{
|
||||
@ -1516,12 +1519,12 @@ var _zcltok_eof_trans []int16 = []int16{
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 0, 390, 391, 392, 394,
|
||||
422, 422, 422, 422, 422, 422, 422, 422,
|
||||
422, 422, 422, 422, 422, 422, 422, 422,
|
||||
422, 422, 422, 422, 422, 422, 422, 422,
|
||||
422, 422, 422, 422, 422, 422, 422, 422,
|
||||
422, 422, 422,
|
||||
255, 255, 255, 0, 391, 392, 393, 395,
|
||||
423, 423, 423, 423, 423, 423, 423, 423,
|
||||
423, 423, 423, 423, 423, 423, 423, 423,
|
||||
423, 423, 423, 423, 423, 423, 423, 423,
|
||||
423, 423, 423, 423, 423, 423, 423, 423,
|
||||
423, 423, 423,
|
||||
}
|
||||
|
||||
const zcltok_start int = 515
|
||||
@ -1539,7 +1542,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
Pos: start,
|
||||
}
|
||||
|
||||
// line 53 "scan_tokens.rl"
|
||||
// line 57 "scan_tokens.rl"
|
||||
|
||||
// Ragel state
|
||||
cs := 0 // Current State
|
||||
@ -1559,8 +1562,16 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
token := func(ty TokenType) {
|
||||
f.emitToken(ty, ts, te)
|
||||
}
|
||||
selfToken := func() {
|
||||
b := data[ts:te]
|
||||
if len(b) != 1 {
|
||||
// should never happen
|
||||
panic("selfToken only works for single-character tokens")
|
||||
}
|
||||
f.emitToken(TokenType(b[0]), ts, te)
|
||||
}
|
||||
|
||||
// line 1570 "scan_tokens.go"
|
||||
// line 1581 "scan_tokens.go"
|
||||
{
|
||||
cs = zcltok_start
|
||||
ts = 0
|
||||
@ -1568,7 +1579,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
act = 0
|
||||
}
|
||||
|
||||
// line 1578 "scan_tokens.go"
|
||||
// line 1589 "scan_tokens.go"
|
||||
{
|
||||
var _klen int
|
||||
var _trans int
|
||||
@ -1590,7 +1601,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
|
||||
ts = p
|
||||
|
||||
// line 1599 "scan_tokens.go"
|
||||
// line 1610 "scan_tokens.go"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1667,87 +1678,94 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
te = p + 1
|
||||
|
||||
case 3:
|
||||
// line 47 "scan_tokens.rl"
|
||||
// line 50 "scan_tokens.rl"
|
||||
|
||||
act = 3
|
||||
case 4:
|
||||
// line 50 "scan_tokens.rl"
|
||||
// line 54 "scan_tokens.rl"
|
||||
|
||||
act = 6
|
||||
act = 7
|
||||
case 5:
|
||||
// line 49 "scan_tokens.rl"
|
||||
// line 51 "scan_tokens.rl"
|
||||
|
||||
te = p + 1
|
||||
{
|
||||
selfToken()
|
||||
}
|
||||
case 6:
|
||||
// line 53 "scan_tokens.rl"
|
||||
|
||||
te = p + 1
|
||||
{
|
||||
token(TokenInvalid)
|
||||
}
|
||||
case 6:
|
||||
// line 50 "scan_tokens.rl"
|
||||
case 7:
|
||||
// line 54 "scan_tokens.rl"
|
||||
|
||||
te = p + 1
|
||||
{
|
||||
token(TokenBadUTF8)
|
||||
}
|
||||
case 7:
|
||||
// line 45 "scan_tokens.rl"
|
||||
case 8:
|
||||
// line 48 "scan_tokens.rl"
|
||||
|
||||
te = p
|
||||
p--
|
||||
|
||||
case 8:
|
||||
// line 46 "scan_tokens.rl"
|
||||
case 9:
|
||||
// line 49 "scan_tokens.rl"
|
||||
|
||||
te = p
|
||||
p--
|
||||
{
|
||||
token(TokenNumberLit)
|
||||
}
|
||||
case 9:
|
||||
// line 47 "scan_tokens.rl"
|
||||
case 10:
|
||||
// line 50 "scan_tokens.rl"
|
||||
|
||||
te = p
|
||||
p--
|
||||
{
|
||||
token(TokenIdent)
|
||||
}
|
||||
case 10:
|
||||
// line 48 "scan_tokens.rl"
|
||||
case 11:
|
||||
// line 52 "scan_tokens.rl"
|
||||
|
||||
te = p
|
||||
p--
|
||||
{
|
||||
token(TokenTabs)
|
||||
}
|
||||
case 11:
|
||||
// line 50 "scan_tokens.rl"
|
||||
case 12:
|
||||
// line 54 "scan_tokens.rl"
|
||||
|
||||
te = p
|
||||
p--
|
||||
{
|
||||
token(TokenBadUTF8)
|
||||
}
|
||||
case 12:
|
||||
// line 46 "scan_tokens.rl"
|
||||
case 13:
|
||||
// line 49 "scan_tokens.rl"
|
||||
|
||||
p = (te) - 1
|
||||
{
|
||||
token(TokenNumberLit)
|
||||
}
|
||||
case 13:
|
||||
// line 47 "scan_tokens.rl"
|
||||
|
||||
p = (te) - 1
|
||||
{
|
||||
token(TokenIdent)
|
||||
}
|
||||
case 14:
|
||||
// line 50 "scan_tokens.rl"
|
||||
|
||||
p = (te) - 1
|
||||
{
|
||||
token(TokenBadUTF8)
|
||||
token(TokenIdent)
|
||||
}
|
||||
case 15:
|
||||
// line 54 "scan_tokens.rl"
|
||||
|
||||
p = (te) - 1
|
||||
{
|
||||
token(TokenBadUTF8)
|
||||
}
|
||||
case 16:
|
||||
// line 1 "NONE"
|
||||
|
||||
switch act {
|
||||
@ -1756,14 +1774,14 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
p = (te) - 1
|
||||
token(TokenIdent)
|
||||
}
|
||||
case 6:
|
||||
case 7:
|
||||
{
|
||||
p = (te) - 1
|
||||
token(TokenBadUTF8)
|
||||
}
|
||||
}
|
||||
|
||||
// line 1749 "scan_tokens.go"
|
||||
// line 1765 "scan_tokens.go"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1779,7 +1797,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
|
||||
ts = 0
|
||||
|
||||
// line 1764 "scan_tokens.go"
|
||||
// line 1780 "scan_tokens.go"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1799,7 +1817,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
|
||||
}
|
||||
|
||||
// line 77 "scan_tokens.rl"
|
||||
// line 89 "scan_tokens.rl"
|
||||
|
||||
// If we fall out here without being in a final state then we've
|
||||
// encountered something that the scanner can't match, which we'll
|
||||
|
||||
@ -34,6 +34,9 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
NumberLit = digit (digit|'.'|('e'|'E') ('+'|'-')? digit)*;
|
||||
Ident = ID_Start ID_Continue*;
|
||||
|
||||
# Symbols that just represent themselves are handled as a single rule.
|
||||
SelfToken = "{" | "}" | "[" | "]" | "(" | ")" | "." | "*" | "/" | "+" | "-" | "=" | "<" | ">" | "!" | "?" | ":" | "\n" | "&" | "|" | "~" | "^" | ";" | "`";
|
||||
|
||||
# Tabs are not valid, but we accept them in the scanner and mark them
|
||||
# as tokens so that we can produce diagnostics advising the user to
|
||||
# use spaces instead.
|
||||
@ -45,6 +48,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
Spaces => {};
|
||||
NumberLit => { token(TokenNumberLit) };
|
||||
Ident => { token(TokenIdent) };
|
||||
SelfToken => { selfToken() };
|
||||
Tabs => { token(TokenTabs) };
|
||||
AnyUTF8 => { token(TokenInvalid) };
|
||||
BrokenUTF8 => { token(TokenBadUTF8) };
|
||||
@ -70,6 +74,14 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
|
||||
token := func (ty TokenType) {
|
||||
f.emitToken(ty, ts, te)
|
||||
}
|
||||
selfToken := func () {
|
||||
b := data[ts:te]
|
||||
if len(b) != 1 {
|
||||
// should never happen
|
||||
panic("selfToken only works for single-character tokens")
|
||||
}
|
||||
f.emitToken(TokenType(b[0]), ts, te)
|
||||
}
|
||||
|
||||
%%{
|
||||
write init;
|
||||
|
||||
@ -213,12 +213,104 @@ func TestScanTokens(t *testing.T) {
|
||||
},
|
||||
},
|
||||
|
||||
// Combinations
|
||||
{
|
||||
` (1 + 2) * 3 `,
|
||||
[]Token{
|
||||
{
|
||||
Type: TokenOParen,
|
||||
Bytes: []byte(`(`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
|
||||
End: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: TokenNumberLit,
|
||||
Bytes: []byte(`1`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 2, Line: 1, Column: 3},
|
||||
End: zcl.Pos{Byte: 3, Line: 1, Column: 4},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: TokenPlus,
|
||||
Bytes: []byte(`+`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
|
||||
End: zcl.Pos{Byte: 5, Line: 1, Column: 6},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: TokenNumberLit,
|
||||
Bytes: []byte(`2`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 6, Line: 1, Column: 7},
|
||||
End: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: TokenCParen,
|
||||
Bytes: []byte(`)`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 7, Line: 1, Column: 8},
|
||||
End: zcl.Pos{Byte: 8, Line: 1, Column: 9},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: TokenStar,
|
||||
Bytes: []byte(`*`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 9, Line: 1, Column: 10},
|
||||
End: zcl.Pos{Byte: 10, Line: 1, Column: 11},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: TokenNumberLit,
|
||||
Bytes: []byte(`3`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 11, Line: 1, Column: 12},
|
||||
End: zcl.Pos{Byte: 12, Line: 1, Column: 13},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: TokenEOF,
|
||||
Bytes: []byte{},
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 13, Line: 1, Column: 14},
|
||||
End: zcl.Pos{Byte: 13, Line: 1, Column: 14},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
// Invalid things
|
||||
{
|
||||
`🌻`,
|
||||
[]Token{
|
||||
{
|
||||
Type: TokenInvalid,
|
||||
Bytes: []byte(`🌻`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
||||
End: zcl.Pos{Byte: 4, Line: 1, Column: 2},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: TokenEOF,
|
||||
Bytes: []byte{},
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 4, Line: 1, Column: 2},
|
||||
End: zcl.Pos{Byte: 4, Line: 1, Column: 2},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
`|`,
|
||||
[]Token{
|
||||
{
|
||||
Type: TokenInvalid,
|
||||
Type: TokenBitwiseOr,
|
||||
Bytes: []byte(`|`),
|
||||
Range: zcl.Range{
|
||||
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
|
||||
|
||||
Loading…
Reference in New Issue
Block a user