go/hcl
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

zclsyntax: detect newlines during scanning

Browse Source 
This is important because our syntax for objects uses newlines as the
separator between items, so this is the only signal we'll get that a
given item has ended and another is beginning.
This commit is contained in:
Martin Atkins 2017-05-29 09:25:21 -07:00
parent 861d0a1b4e
commit f1c91b8eea
3 changed files with 415 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

646
zcl/zclsyntax/scan_tokens.go
View File

@ -23,10 +23,10 @@ var _zcltok_actions []byte = []byte{
1, 44, 1, 45, 1, 46, 1, 47,
1, 48, 1, 49, 1, 50, 1, 51,
1, 52, 1, 53, 1, 54, 1, 55,
1, 56, 1, 57, 2, 0, 1, 2,
3, 16, 2, 3, 17, 2, 3, 28,
2, 3, 29, 2, 3, 30, 2, 3,
31, 2, 3, 32,
1, 56, 1, 57, 1, 58, 1, 59,
2, 0, 1, 2, 3, 16, 2, 3,
17, 2, 3, 28, 2, 3, 29, 2,
3, 30, 2, 3, 31, 2, 3, 32,
}
var _zcltok_key_offsets []int16 = []int16{
@ -147,16 +147,16 @@ var _zcltok_key_offsets []int16 = []int16{
5828, 5841, 5845, 5860, 5889, 5900, 5902, 5906,
5910, 5915, 5919, 5921, 5928, 5932, 5940, 5944,
5945, 5947, 5949, 5951, 5953, 5955, 5956, 5957,
5959, 5961, 5963, 5964, 5965, 6038, 6039, 6040,
6041, 6042, 6043, 6045, 6046, 6051, 6053, 6054,
6097, 6098, 6099, 6101, 6106, 6110, 6110, 6112,
6114, 6125, 6135, 6143, 6144, 6146, 6147, 6151,
6155, 6165, 6169, 6176, 6187, 6194, 6198, 6204,
6215, 6247, 6296, 6311, 6326, 6331, 6333, 6338,
6370, 6378, 6380, 6402, 6424, 6426, 6442, 6458,
6473, 6482, 6496, 6510, 6526, 6527, 6528, 6529,
6530, 6532, 6534, 6536, 6550, 6564, 6565, 6566,
6568, 6570,
5959, 5961, 5963, 5964, 5965, 6039, 6040, 6041,
6042, 6043, 6044, 6045, 6047, 6048, 6053, 6055,
6056, 6099, 6100, 6101, 6103, 6108, 6112, 6112,
6114, 6116, 6127, 6137, 6145, 6146, 6148, 6149,
6153, 6157, 6167, 6171, 6178, 6189, 6196, 6200,
6206, 6217, 6249, 6298, 6313, 6328, 6333, 6335,
6340, 6372, 6380, 6382, 6404, 6426, 6428, 6444,
6460, 6475, 6484, 6498, 6512, 6528, 6529, 6530,
6531, 6532, 6534, 6536, 6538, 6552, 6566, 6567,
6568, 6570, 6572,
}
var _zcltok_trans_keys []byte = []byte{
@ -905,83 +905,83 @@ var _zcltok_trans_keys []byte = []byte{
161, 167, 169, 191, 158, 191, 192, 255,
123, 128, 191, 128, 191, 128, 191, 128,
191, 128, 191, 10, 123, 128, 191, 128,
191, 128, 191, 123, 123, 9, 10, 32,
33, 34, 35, 38, 44, 47, 60, 62,
64, 92, 95, 123, 124, 125, 126, 127,
194, 195, 198, 199, 203, 204, 205, 206,
207, 210, 212, 213, 214, 215, 216, 217,
219, 220, 221, 222, 223, 224, 225, 226,
227, 228, 233, 234, 237, 238, 239, 240,
0, 39, 40, 46, 48, 57, 58, 63,
65, 90, 91, 96, 97, 122, 192, 193,
196, 218, 229, 236, 241, 247, 9, 32,
61, 10, 38, 42, 47, 42, 46, 69,
101, 48, 57, 60, 61, 61, 95, 194,
195, 198, 199, 203, 204, 205, 206, 207,
210, 212, 213, 214, 215, 216, 217, 219,
220, 221, 222, 223, 224, 225, 226, 227,
228, 233, 234, 237, 239, 240, 243, 48,
57, 65, 90, 97, 122, 196, 218, 229,
236, 124, 125, 128, 191, 170, 181, 186,
128, 191, 151, 183, 128, 255, 192, 255,
0, 127, 173, 130, 133, 146, 159, 165,
171, 175, 191, 192, 255, 181, 190, 128,
175, 176, 183, 184, 185, 186, 191, 134,
139, 141, 162, 128, 135, 136, 255, 182,
130, 137, 176, 151, 152, 154, 160, 136,
191, 192, 255, 128, 143, 144, 170, 171,
175, 176, 178, 179, 191, 128, 159, 160,
191, 176, 128, 138, 139, 173, 174, 255,
148, 150, 164, 167, 173, 176, 185, 189,
190, 192, 255, 144, 128, 145, 146, 175,
176, 191, 128, 140, 141, 255, 166, 176,
178, 191, 192, 255, 186, 128, 137, 138,
170, 171, 179, 180, 181, 182, 191, 160,
161, 162, 164, 165, 166, 167, 168, 169,
170, 171, 172, 173, 174, 175, 176, 177,
178, 179, 180, 181, 182, 183, 184, 185,
186, 187, 188, 189, 190, 128, 191, 128,
129, 130, 131, 137, 138, 139, 140, 141,
142, 143, 144, 153, 154, 155, 156, 157,
158, 159, 160, 161, 162, 163, 164, 165,
166, 167, 168, 169, 170, 171, 172, 173,
174, 175, 176, 177, 178, 179, 180, 182,
183, 184, 188, 189, 190, 191, 132, 187,
129, 130, 132, 133, 134, 176, 177, 178,
179, 180, 181, 182, 183, 128, 191, 128,
129, 130, 131, 132, 133, 134, 135, 144,
136, 143, 145, 191, 192, 255, 182, 183,
184, 128, 191, 128, 191, 191, 128, 190,
192, 255, 128, 146, 147, 148, 152, 153,
154, 155, 156, 158, 159, 160, 161, 162,
163, 164, 165, 166, 167, 168, 169, 170,
171, 172, 173, 174, 175, 176, 129, 191,
192, 255, 158, 159, 128, 157, 160, 191,
192, 255, 128, 191, 164, 169, 171, 172,
173, 174, 175, 180, 181, 182, 183, 184,
185, 187, 188, 189, 190, 191, 128, 163,
165, 186, 144, 145, 146, 147, 148, 150,
151, 152, 155, 157, 158, 160, 170, 171,
172, 175, 128, 159, 161, 169, 173, 191,
128, 191, 10, 13, 33, 34, 36, 92,
128, 191, 192, 223, 224, 239, 240, 247,
191, 128, 191, 123, 123, 9, 10, 13,
32, 33, 34, 35, 38, 44, 47, 60,
62, 64, 92, 95, 123, 124, 125, 126,
127, 194, 195, 198, 199, 203, 204, 205,
206, 207, 210, 212, 213, 214, 215, 216,
217, 219, 220, 221, 222, 223, 224, 225,
226, 227, 228, 233, 234, 237, 238, 239,
240, 0, 39, 40, 46, 48, 57, 58,
63, 65, 90, 91, 96, 97, 122, 192,
193, 196, 218, 229, 236, 241, 247, 9,
10, 32, 61, 10, 38, 42, 47, 42,
46, 69, 101, 48, 57, 60, 61, 61,
95, 194, 195, 198, 199, 203, 204, 205,
206, 207, 210, 212, 213, 214, 215, 216,
217, 219, 220, 221, 222, 223, 224, 225,
226, 227, 228, 233, 234, 237, 239, 240,
243, 48, 57, 65, 90, 97, 122, 196,
218, 229, 236, 124, 125, 128, 191, 170,
181, 186, 128, 191, 151, 183, 128, 255,
192, 255, 0, 127, 173, 130, 133, 146,
159, 165, 171, 175, 191, 192, 255, 181,
190, 128, 175, 176, 183, 184, 185, 186,
191, 134, 139, 141, 162, 128, 135, 136,
255, 182, 130, 137, 176, 151, 152, 154,
160, 136, 191, 192, 255, 128, 143, 144,
170, 171, 175, 176, 178, 179, 191, 128,
159, 160, 191, 176, 128, 138, 139, 173,
174, 255, 148, 150, 164, 167, 173, 176,
185, 189, 190, 192, 255, 144, 128, 145,
146, 175, 176, 191, 128, 140, 141, 255,
166, 176, 178, 191, 192, 255, 186, 128,
137, 138, 170, 171, 179, 180, 181, 182,
191, 160, 161, 162, 164, 165, 166, 167,
168, 169, 170, 171, 172, 173, 174, 175,
176, 177, 178, 179, 180, 181, 182, 183,
184, 185, 186, 187, 188, 189, 190, 128,
191, 128, 129, 130, 131, 137, 138, 139,
140, 141, 142, 143, 144, 153, 154, 155,
156, 157, 158, 159, 160, 161, 162, 163,
164, 165, 166, 167, 168, 169, 170, 171,
172, 173, 174, 175, 176, 177, 178, 179,
180, 182, 183, 184, 188, 189, 190, 191,
132, 187, 129, 130, 132, 133, 134, 176,
177, 178, 179, 180, 181, 182, 183, 128,
191, 128, 129, 130, 131, 132, 133, 134,
135, 144, 136, 143, 145, 191, 192, 255,
182, 183, 184, 128, 191, 128, 191, 191,
128, 190, 192, 255, 128, 146, 147, 148,
152, 153, 154, 155, 156, 158, 159, 160,
161, 162, 163, 164, 165, 166, 167, 168,
169, 170, 171, 172, 173, 174, 175, 176,
129, 191, 192, 255, 158, 159, 128, 157,
160, 191, 192, 255, 128, 191, 164, 169,
171, 172, 173, 174, 175, 180, 181, 182,
183, 184, 185, 187, 188, 189, 190, 191,
128, 163, 165, 186, 144, 145, 146, 147,
148, 150, 151, 152, 155, 157, 158, 160,
170, 171, 172, 175, 128, 159, 161, 169,
173, 191, 128, 191, 10, 13, 33, 34,
36, 92, 128, 191, 192, 223, 224, 239,
240, 247, 248, 255, 10, 13, 33, 34,
36, 92, 128, 191, 192, 223, 224, 239,
240, 247, 248, 255, 10, 13, 33, 36,
92, 128, 191, 192, 223, 224, 239, 240,
247, 248, 255, 33, 36, 92, 192, 223,
224, 239, 240, 247, 10, 13, 33, 34,
36, 92, 192, 223, 224, 239, 240, 247,
248, 255, 10, 13, 33, 34, 36, 92,
128, 191, 192, 223, 224, 239, 240, 247,
248, 255, 10, 13, 33, 36, 92, 128,
191, 192, 223, 224, 239, 240, 247, 248,
255, 33, 36, 92, 192, 223, 224, 239,
240, 247, 10, 13, 33, 34, 36, 92,
128, 223, 224, 239, 240, 247, 248, 255,
10, 13, 33, 34, 36, 92, 128, 191,
192, 223, 224, 239, 240, 247, 248, 255,
123, 126, 123, 126, 128, 191, 128, 191,
128, 191, 10, 13, 33, 36, 128, 191,
192, 223, 224, 239, 240, 247, 248, 255,
10, 13, 33, 34, 36, 92, 128, 223,
224, 239, 240, 247, 248, 255, 10, 13,
33, 34, 36, 92, 128, 191, 192, 223,
224, 239, 240, 247, 248, 255, 123, 126,
123, 126, 128, 191, 128, 191, 128, 191,
10, 13, 33, 36, 128, 191, 192, 223,
224, 239, 240, 247, 248, 255, 10, 13,
33, 36, 128, 191, 192, 223, 224, 239,
240, 247, 248, 255, 126, 126, 128, 191,
128, 191, 128, 191,
224, 239, 240, 247, 248, 255, 126, 126,
128, 191, 128, 191, 128, 191,
}
var _zcltok_single_lengths []byte = []byte{
@ -1102,16 +1102,16 @@ var _zcltok_single_lengths []byte = []byte{
7, 0, 7, 17, 3, 0, 2, 0,
3, 0, 0, 1, 0, 2, 0, 1,
0, 0, 0, 0, 0, 1, 1, 0,
0, 0, 1, 1, 51, 1, 1, 1,
1, 1, 2, 1, 3, 2, 1, 33,
1, 1, 0, 3, 2, 0, 0, 0,
1, 2, 4, 1, 0, 1, 0, 0,
0, 0, 1, 1, 1, 0, 0, 1,
30, 47, 13, 9, 3, 0, 1, 28,
2, 0, 18, 16, 0, 6, 6, 5,
3, 6, 6, 6, 1, 1, 1, 1,
0, 0, 0, 4, 4, 1, 1, 0,
0, 0,
0, 0, 1, 1, 52, 1, 1, 1,
1, 1, 1, 2, 1, 3, 2, 1,
33, 1, 1, 0, 3, 2, 0, 0,
0, 1, 2, 4, 1, 0, 1, 0,
0, 0, 0, 1, 1, 1, 0, 0,
1, 30, 47, 13, 9, 3, 0, 1,
28, 2, 0, 18, 16, 0, 6, 6,
5, 3, 6, 6, 6, 1, 1, 1,
1, 0, 0, 0, 4, 4, 1, 1,
0, 0, 0,
}
var _zcltok_range_lengths []byte = []byte{
@ -1233,15 +1233,15 @@ var _zcltok_range_lengths []byte = []byte{
1, 2, 1, 3, 2, 3, 2, 0,
1, 1, 1, 1, 1, 0, 0, 1,
1, 1, 0, 0, 11, 0, 0, 0,
0, 0, 0, 0, 1, 0, 0, 5,
0, 0, 1, 1, 1, 0, 1, 1,
5, 4, 2, 0, 1, 0, 2, 2,
5, 2, 3, 5, 3, 2, 3, 5,
1, 1, 1, 3, 1, 1, 2, 2,
3, 1, 2, 3, 1, 5, 5, 5,
3, 4, 4, 5, 0, 0, 0, 0,
1, 1, 1, 5, 5, 0, 0, 1,
1, 1,
0, 0, 0, 0, 0, 1, 0, 0,
5, 0, 0, 1, 1, 1, 0, 1,
1, 5, 4, 2, 0, 1, 0, 2,
2, 5, 2, 3, 5, 3, 2, 3,
5, 1, 1, 1, 3, 1, 1, 2,
2, 3, 1, 2, 3, 1, 5, 5,
5, 3, 4, 4, 5, 0, 0, 0,
0, 1, 1, 1, 5, 5, 0, 0,
1, 1, 1,
}
var _zcltok_index_offsets []int16 = []int16{
@ -1362,16 +1362,16 @@ var _zcltok_index_offsets []int16 = []int16{
4624, 4635, 4638, 4650, 4674, 4682, 4684, 4688,
4691, 4696, 4699, 4701, 4706, 4709, 4715, 4718,
4720, 4722, 4724, 4726, 4728, 4730, 4732, 4734,
4736, 4738, 4740, 4742, 4744, 4807, 4809, 4811,
4813, 4815, 4817, 4820, 4822, 4827, 4830, 4832,
4871, 4873, 4875, 4877, 4882, 4886, 4887, 4889,
4891, 4898, 4905, 4912, 4914, 4916, 4918, 4921,
4924, 4930, 4933, 4938, 4945, 4950, 4953, 4957,
4964, 4996, 5045, 5060, 5073, 5078, 5080, 5084,
5115, 5121, 5123, 5144, 5164, 5166, 5178, 5190,
5201, 5208, 5219, 5230, 5242, 5244, 5246, 5248,
5250, 5252, 5254, 5256, 5266, 5276, 5278, 5280,
5282, 5284,
4736, 4738, 4740, 4742, 4744, 4808, 4810, 4812,
4814, 4816, 4818, 4820, 4823, 4825, 4830, 4833,
4835, 4874, 4876, 4878, 4880, 4885, 4889, 4890,
4892, 4894, 4901, 4908, 4915, 4917, 4919, 4921,
4924, 4927, 4933, 4936, 4941, 4948, 4953, 4956,
4960, 4967, 4999, 5048, 5063, 5076, 5081, 5083,
5087, 5118, 5124, 5126, 5147, 5167, 5169, 5181,
5193, 5204, 5211, 5222, 5233, 5245, 5247, 5249,
5251, 5253, 5255, 5257, 5259, 5269, 5279, 5281,
5283, 5285, 5287,
}
var _zcltok_indicies []int16 = []int16{
@ -1968,79 +1968,80 @@ var _zcltok_indicies []int16 = []int16{
766, 765, 767, 765, 768, 765, 766, 769,
770, 769, 772, 771, 773, 774, 774, 771,
775, 771, 776, 773, 777, 774, 778, 774,
779, 780, 781, 782, 783, 784, 785, 669,
786, 787, 788, 669, 669, 669, 789, 790,
791, 792, 669, 795, 796, 798, 799, 800,
794, 801, 802, 803, 804, 805, 806, 807,
808, 809, 810, 811, 812, 813, 814, 815,
816, 817, 818, 819, 820, 822, 823, 824,
825, 826, 827, 669, 780, 8, 780, 419,
780, 419, 794, 797, 821, 828, 793, 779,
829, 781, 830, 832, 831, 2, 1, 833,
831, 3, 1, 831, 4, 3, 8, 836,
836, 8, 835, 837, 838, 831, 839, 831,
419, 841, 842, 488, 469, 843, 469, 844,
845, 846, 847, 848, 849, 850, 851, 852,
853, 854, 543, 855, 519, 856, 857, 858,
779, 780, 781, 782, 783, 784, 785, 786,
669, 788, 789, 790, 669, 669, 669, 791,
792, 793, 794, 669, 797, 798, 800, 801,
802, 796, 803, 804, 805, 806, 807, 808,
809, 810, 811, 812, 813, 814, 815, 816,
817, 818, 819, 820, 821, 822, 824, 825,
826, 827, 828, 829, 669, 787, 8, 787,
419, 787, 419, 796, 799, 823, 830, 795,
779, 831, 780, 832, 782, 833, 835, 834,
2, 1, 836, 834, 3, 1, 834, 4,
3, 8, 839, 839, 8, 838, 840, 841,
834, 842, 834, 419, 844, 845, 488, 469,
846, 469, 847, 848, 849, 850, 851, 852,
853, 854, 855, 856, 857, 543, 858, 519,
859, 860, 861, 862, 863, 864, 865, 866,
867, 419, 419, 419, 424, 564, 840, 868,
831, 869, 831, 669, 870, 419, 419, 419,
669, 870, 669, 669, 419, 870, 419, 870,
419, 870, 419, 669, 669, 669, 669, 669,
870, 419, 669, 669, 669, 419, 669, 419,
870, 419, 669, 669, 669, 669, 419, 870,
669, 419, 669, 419, 669, 419, 669, 669,
419, 669, 870, 419, 669, 419, 669, 419,
669, 870, 669, 419, 870, 669, 419, 669,
419, 870, 669, 669, 669, 669, 669, 870,
419, 419, 669, 419, 669, 870, 669, 419,
870, 669, 669, 870, 419, 419, 669, 419,
669, 419, 669, 870, 871, 872, 873, 874,
875, 876, 877, 878, 879, 880, 881, 714,
882, 883, 884, 885, 886, 887, 888, 889,
890, 891, 892, 893, 892, 894, 895, 896,
897, 898, 670, 870, 899, 900, 901, 902,
867, 868, 869, 870, 419, 419, 419, 424,
564, 843, 871, 834, 872, 834, 669, 873,
419, 419, 419, 669, 873, 669, 669, 419,
873, 419, 873, 419, 873, 419, 669, 669,
669, 669, 669, 873, 419, 669, 669, 669,
419, 669, 419, 873, 419, 669, 669, 669,
669, 419, 873, 669, 419, 669, 419, 669,
419, 669, 669, 419, 669, 873, 419, 669,
419, 669, 419, 669, 873, 669, 419, 873,
669, 419, 669, 419, 873, 669, 669, 669,
669, 669, 873, 419, 419, 669, 419, 669,
873, 669, 419, 873, 669, 669, 873, 419,
419, 669, 419, 669, 419, 669, 873, 874,
875, 876, 877, 878, 879, 880, 881, 882,
883, 884, 714, 885, 886, 887, 888, 889,
890, 891, 892, 893, 894, 895, 896, 895,
897, 898, 899, 900, 901, 670, 873, 902,
903, 904, 905, 906, 907, 908, 909, 910,
911, 912, 913, 914, 915, 916, 917, 724,
918, 919, 920, 691, 921, 922, 923, 924,
925, 926, 670, 927, 928, 929, 930, 931,
932, 933, 934, 673, 935, 670, 673, 936,
937, 938, 939, 682, 870, 940, 941, 942,
943, 702, 944, 945, 682, 946, 947, 948,
949, 950, 670, 870, 951, 910, 952, 953,
954, 682, 955, 956, 673, 670, 682, 424,
870, 920, 670, 673, 682, 424, 682, 424,
957, 682, 870, 424, 673, 958, 959, 673,
960, 961, 680, 962, 963, 964, 965, 966,
916, 967, 968, 969, 970, 971, 972, 973,
974, 975, 976, 977, 978, 935, 979, 673,
682, 424, 870, 980, 981, 682, 670, 870,
424, 670, 870, 673, 982, 730, 983, 984,
985, 986, 987, 988, 989, 990, 670, 991,
992, 993, 994, 995, 996, 670, 682, 870,
998, 999, 1000, 1001, 1002, 1003, 1004, 1005,
1006, 1007, 1008, 1004, 1010, 1011, 1012, 1013,
997, 1009, 997, 870, 997, 870, 1014, 1014,
1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022,
1019, 766, 1023, 1023, 1024, 1023, 1024, 1018,
1023, 767, 768, 1025, 1023, 766, 1023, 1023,
1026, 1026, 1018, 1023, 767, 768, 1025, 1023,
766, 1026, 1026, 1018, 1027, 1028, 1029, 766,
1023, 1023, 1024, 1023, 1024, 1018, 767, 768,
1025, 1023, 766, 1023, 1023, 1024, 1023, 1024,
1018, 767, 768, 1025, 1023, 766, 1023, 1023,
1024, 1023, 1024, 1018, 768, 767, 768, 1025,
1023, 766, 1031, 766, 1033, 1032, 1034, 766,
1036, 1035, 766, 1037, 770, 1037, 1038, 1037,
772, 1039, 1040, 1041, 1042, 1043, 1044, 1045,
1042, 774, 772, 1039, 1047, 1047, 1046, 775,
776, 1048, 1046, 774, 1050, 1049, 1052, 1051,
774, 1053, 775, 1053, 776, 1053,
911, 912, 913, 914, 915, 916, 917, 918,
919, 920, 724, 921, 922, 923, 691, 924,
925, 926, 927, 928, 929, 670, 930, 931,
932, 933, 934, 935, 936, 937, 673, 938,
670, 673, 939, 940, 941, 942, 682, 873,
943, 944, 945, 946, 702, 947, 948, 682,
949, 950, 951, 952, 953, 670, 873, 954,
913, 955, 956, 957, 682, 958, 959, 673,
670, 682, 424, 873, 923, 670, 673, 682,
424, 682, 424, 960, 682, 873, 424, 673,
961, 962, 673, 963, 964, 680, 965, 966,
967, 968, 969, 919, 970, 971, 972, 973,
974, 975, 976, 977, 978, 979, 980, 981,
938, 982, 673, 682, 424, 873, 983, 984,
682, 670, 873, 424, 670, 873, 673, 985,
730, 986, 987, 988, 989, 990, 991, 992,
993, 670, 994, 995, 996, 997, 998, 999,
670, 682, 873, 1001, 1002, 1003, 1004, 1005,
1006, 1007, 1008, 1009, 1010, 1011, 1007, 1013,
1014, 1015, 1016, 1000, 1012, 1000, 873, 1000,
873, 1017, 1017, 1018, 1019, 1020, 1021, 1022,
1023, 1024, 1025, 1022, 766, 1026, 1026, 1027,
1026, 1027, 1021, 1026, 767, 768, 1028, 1026,
766, 1026, 1026, 1029, 1029, 1021, 1026, 767,
768, 1028, 1026, 766, 1029, 1029, 1021, 1030,
1031, 1032, 766, 1026, 1026, 1027, 1026, 1027,
1021, 767, 768, 1028, 1026, 766, 1026, 1026,
1027, 1026, 1027, 1021, 767, 768, 1028, 1026,
766, 1026, 1026, 1027, 1026, 1027, 1021, 768,
767, 768, 1028, 1026, 766, 1034, 766, 1036,
1035, 1037, 766, 1039, 1038, 766, 1040, 770,
1040, 1041, 1040, 772, 1042, 1043, 1044, 1045,
1046, 1047, 1048, 1045, 774, 772, 1042, 1050,
1050, 1049, 775, 776, 1051, 1049, 774, 1053,
1052, 1055, 1054, 774, 1056, 775, 1056, 776,
1056,
}
var _zcltok_trans_targs []int16 = []int16{
940, 1, 944, 2, 3, 947, 940, 5,
948, 940, 7, 8, 285, 11, 12, 13,
940, 1, 945, 2, 3, 948, 940, 5,
949, 940, 7, 8, 285, 11, 12, 13,
14, 15, 286, 287, 18, 288, 20, 21,
289, 290, 291, 292, 293, 294, 295, 296,
297, 298, 327, 347, 352, 126, 127, 128,
@ -2091,7 +2092,7 @@ var _zcltok_trans_targs []int16 = []int16{
384, 385, 386, 387, 388, 389, 390, 391,
392, 393, 394, 395, 396, 397, 398, 399,
400, 401, 402, 404, 405, 406, 407, 409,
411, 413, 940, 951, 436, 437, 438, 439,
411, 413, 940, 952, 436, 437, 438, 439,
416, 440, 441, 442, 443, 444, 445, 446,
447, 448, 449, 450, 451, 452, 453, 454,
455, 456, 457, 458, 459, 460, 461, 462,
@ -2134,52 +2135,53 @@ var _zcltok_trans_targs []int16 = []int16{
888, 889, 890, 891, 894, 895, 897, 898,
899, 901, 902, 903, 904, 905, 906, 907,
908, 909, 910, 911, 913, 914, 915, 916,
919, 921, 922, 924, 926, 989, 990, 928,
929, 989, 931, 1003, 1003, 1003, 1004, 935,
936, 1005, 1006, 941, 940, 942, 943, 940,
944, 945, 946, 949, 950, 940, 952, 940,
953, 940, 954, 955, 956, 957, 958, 959,
960, 961, 962, 963, 964, 965, 966, 967,
968, 969, 970, 971, 972, 973, 974, 975,
976, 977, 978, 979, 980, 981, 982, 983,
984, 985, 986, 987, 988, 940, 940, 940,
940, 940, 940, 940, 4, 6, 940, 940,
940, 414, 415, 419, 420, 421, 422, 423,
424, 425, 426, 427, 428, 429, 430, 432,
434, 435, 467, 508, 523, 530, 532, 534,
554, 557, 573, 686, 940, 940, 940, 690,
691, 692, 693, 694, 695, 696, 697, 698,
699, 700, 702, 703, 704, 705, 706, 707,
708, 709, 710, 711, 712, 713, 714, 715,
716, 717, 718, 719, 720, 721, 722, 724,
725, 726, 727, 728, 729, 730, 731, 732,
733, 734, 735, 736, 737, 738, 740, 741,
742, 744, 745, 746, 747, 748, 749, 750,
751, 752, 753, 754, 755, 756, 757, 759,
760, 761, 762, 763, 764, 765, 766, 767,
769, 770, 771, 772, 773, 774, 775, 776,
777, 778, 779, 780, 781, 782, 783, 784,
785, 786, 788, 789, 790, 791, 792, 793,
794, 795, 796, 797, 798, 799, 800, 801,
802, 803, 804, 805, 806, 807, 808, 810,
811, 812, 813, 814, 815, 816, 817, 818,
819, 820, 821, 822, 823, 824, 825, 854,
879, 882, 883, 885, 892, 893, 896, 900,
912, 917, 918, 920, 923, 925, 989, 996,
989, 998, 991, 989, 1000, 1001, 1002, 989,
927, 930, 992, 993, 994, 995, 989, 997,
989, 989, 999, 989, 989, 989, 932, 933,
938, 939, 1003, 1007, 1008, 1009, 1003, 934,
937, 1003, 1003, 1003, 1003, 1003,
919, 921, 922, 924, 926, 990, 991, 928,
929, 990, 931, 1004, 1004, 1004, 1005, 935,
936, 1006, 1007, 941, 940, 942, 943, 944,
940, 945, 946, 940, 947, 950, 951, 940,
953, 940, 954, 940, 955, 956, 957, 958,
959, 960, 961, 962, 963, 964, 965, 966,
967, 968, 969, 970, 971, 972, 973, 974,
975, 976, 977, 978, 979, 980, 981, 982,
983, 984, 985, 986, 987, 988, 989, 940,
940, 940, 940, 940, 940, 940, 940, 4,
6, 940, 940, 940, 414, 415, 419, 420,
421, 422, 423, 424, 425, 426, 427, 428,
429, 430, 432, 434, 435, 467, 508, 523,
530, 532, 534, 554, 557, 573, 686, 940,
940, 940, 690, 691, 692, 693, 694, 695,
696, 697, 698, 699, 700, 702, 703, 704,
705, 706, 707, 708, 709, 710, 711, 712,
713, 714, 715, 716, 717, 718, 719, 720,
721, 722, 724, 725, 726, 727, 728, 729,
730, 731, 732, 733, 734, 735, 736, 737,
738, 740, 741, 742, 744, 745, 746, 747,
748, 749, 750, 751, 752, 753, 754, 755,
756, 757, 759, 760, 761, 762, 763, 764,
765, 766, 767, 769, 770, 771, 772, 773,
774, 775, 776, 777, 778, 779, 780, 781,
782, 783, 784, 785, 786, 788, 789, 790,
791, 792, 793, 794, 795, 796, 797, 798,
799, 800, 801, 802, 803, 804, 805, 806,
807, 808, 810, 811, 812, 813, 814, 815,
816, 817, 818, 819, 820, 821, 822, 823,
824, 825, 854, 879, 882, 883, 885, 892,
893, 896, 900, 912, 917, 918, 920, 923,
925, 990, 997, 990, 999, 992, 990, 1001,
1002, 1003, 990, 927, 930, 993, 994, 995,
996, 990, 998, 990, 990, 1000, 990, 990,
990, 932, 933, 938, 939, 1004, 1008, 1009,
1010, 1004, 934, 937, 1004, 1004, 1004, 1004,
1004,
}
var _zcltok_trans_actions []byte = []byte{
99, 0, 113, 0, 0, 113, 91, 0,
5, 95, 0, 0, 0, 0, 0, 0,
103, 0, 117, 0, 0, 117, 95, 0,
5, 99, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 71, 0, 0, 0,
0, 0, 0, 0, 73, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
@ -2226,7 +2228,7 @@ var _zcltok_trans_actions []byte = []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 93, 110, 0, 0, 0, 0,
0, 0, 97, 114, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
@ -2257,7 +2259,7 @@ var _zcltok_trans_actions []byte = []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 97, 73, 0, 0,
0, 0, 0, 0, 101, 75, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
@ -2270,19 +2272,20 @@ var _zcltok_trans_actions []byte = []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 27, 5, 0,
0, 29, 0, 49, 35, 47, 104, 0,
0, 0, 0, 0, 61, 0, 0, 69,
119, 0, 116, 5, 0, 63, 0, 65,
0, 75, 0, 0, 0, 0, 0, 0,
0, 29, 0, 49, 35, 47, 108, 0,
0, 0, 0, 0, 51, 0, 0, 0,
71, 123, 0, 63, 120, 5, 0, 65,
0, 67, 0, 77, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
5, 5, 5, 122, 122, 122, 122, 122,
122, 5, 5, 122, 5, 87, 77, 85,
51, 57, 83, 79, 0, 0, 55, 53,
81, 0, 0, 0, 0, 0, 0, 0,
0, 0, 5, 5, 5, 126, 126, 126,
126, 126, 126, 5, 5, 126, 5, 89,
91, 79, 87, 53, 59, 85, 81, 0,
0, 57, 55, 83, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 59, 67, 89, 0,
0, 0, 0, 0, 0, 0, 0, 61,
69, 93, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
@ -2300,12 +2303,12 @@ var _zcltok_trans_actions []byte = []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 13, 0,
11, 0, 5, 15, 0, 5, 5, 21,
0, 0, 0, 5, 5, 5, 23, 0,
19, 9, 0, 17, 7, 25, 0, 0,
0, 0, 37, 0, 107, 107, 43, 0,
0, 41, 33, 39, 31, 45,
0, 13, 0, 11, 0, 5, 15, 0,
5, 5, 21, 0, 0, 0, 5, 5,
5, 23, 0, 19, 9, 0, 17, 7,
25, 0, 0, 0, 0, 37, 0, 111,
111, 43, 0, 0, 41, 33, 39, 31,
45,
}
var _zcltok_to_state_actions []byte = []byte{
@ -2432,10 +2435,10 @@ var _zcltok_to_state_actions []byte = []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 1, 0, 0,
0, 0, 0, 0, 0, 0, 1, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 101, 0, 0, 0, 0,
0, 0,
0, 0, 0, 0, 105, 0, 0, 0,
0, 0, 0,
}
var _zcltok_from_state_actions []byte = []byte{
@ -2562,10 +2565,10 @@ var _zcltok_from_state_actions []byte = []byte{
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 3, 0, 0,
0, 0, 0, 0, 0, 0, 3, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 3, 0, 0, 0, 0,
0, 0,
0, 0, 0, 0, 3, 0, 0, 0,
0, 0, 0,
}
var _zcltok_eof_trans []int16 = []int16{
@ -2686,24 +2689,24 @@ var _zcltok_eof_trans []int16 = []int16{
669, 669, 669, 669, 669, 669, 669, 669,
669, 669, 669, 669, 669, 669, 669, 766,
766, 766, 766, 770, 770, 772, 774, 772,
772, 774, 0, 0, 0, 830, 831, 832,
1, 832, 832, 835, 836, 832, 832, 841,
832, 832, 871, 871, 871, 871, 871, 871,
871, 871, 871, 871, 871, 871, 871, 871,
871, 871, 871, 871, 871, 871, 871, 871,
871, 871, 871, 871, 871, 871, 871, 871,
871, 871, 871, 871, 871, 0, 1024, 1024,
1024, 1024, 1024, 1024, 1031, 1033, 1031, 1036,
1038, 1038, 1038, 0, 1047, 1050, 1052, 1054,
1054, 1054,
772, 774, 0, 0, 0, 832, 833, 834,
835, 1, 835, 835, 838, 839, 835, 835,
844, 835, 835, 874, 874, 874, 874, 874,
874, 874, 874, 874, 874, 874, 874, 874,
874, 874, 874, 874, 874, 874, 874, 874,
874, 874, 874, 874, 874, 874, 874, 874,
874, 874, 874, 874, 874, 874, 0, 1027,
1027, 1027, 1027, 1027, 1027, 1034, 1036, 1034,
1039, 1041, 1041, 1041, 0, 1050, 1053, 1055,
1057, 1057, 1057,
}
const zcltok_start int = 940
const zcltok_first_final int = 940
const zcltok_error int = 0
const zcltok_en_stringTemplate int = 989
const zcltok_en_heredocTemplate int = 1003
const zcltok_en_stringTemplate int = 990
const zcltok_en_heredocTemplate int = 1004
const zcltok_en_main int = 940
// line 15 "scan_tokens.rl"
@ -2715,7 +2718,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
Pos: start,
}
// line 237 "scan_tokens.rl"
// line 238 "scan_tokens.rl"
// Ragel state
p := 0 // "Pointer" into data
@ -2743,7 +2746,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
var retBraces []int // stack of brace levels that cause us to use fret
var heredocs []heredocInProgress // stack of heredocs we're currently processing
// line 272 "scan_tokens.rl"
// line 273 "scan_tokens.rl"
// Make Go compiler happy
_ = ts
@ -2763,7 +2766,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
f.emitToken(TokenType(b[0]), ts, te)
}
// line 2775 "scan_tokens.go"
// line 2778 "scan_tokens.go"
{
top = 0
ts = 0
@ -2771,7 +2774,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
act = 0
}
// line 2783 "scan_tokens.go"
// line 2786 "scan_tokens.go"
{
var _klen int
var _trans int
@ -2796,7 +2799,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
ts = p
// line 2807 "scan_tokens.go"
// line 2810 "scan_tokens.go"
}
}
@ -3203,60 +3206,67 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
act = 15
case 30:
// line 222 "scan_tokens.rl"
// line 223 "scan_tokens.rl"
act = 21
act = 22
case 31:
// line 233 "scan_tokens.rl"
act = 28
case 32:
// line 234 "scan_tokens.rl"
act = 29
case 32:
// line 235 "scan_tokens.rl"
act = 30
case 33:
// line 217 "scan_tokens.rl"
// line 216 "scan_tokens.rl"
te = p + 1
{
token(TokenNotEqual)
token(TokenNewline)
}
case 34:
// line 218 "scan_tokens.rl"
te = p + 1
{
token(TokenGreaterThanEq)
token(TokenNotEqual)
}
case 35:
// line 219 "scan_tokens.rl"
te = p + 1
{
token(TokenLessThanEq)
token(TokenGreaterThanEq)
}
case 36:
// line 220 "scan_tokens.rl"
te = p + 1
{
token(TokenAnd)
token(TokenLessThanEq)
}
case 37:
// line 221 "scan_tokens.rl"
te = p + 1
{
token(TokenOr)
token(TokenAnd)
}
case 38:
// line 222 "scan_tokens.rl"
te = p + 1
{
selfToken()
token(TokenOr)
}
case 39:
// line 223 "scan_tokens.rl"
te = p + 1
{
selfToken()
}
case 40:
// line 146 "scan_tokens.rl"
te = p + 1
@ -3264,7 +3274,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
token(TokenOBrace)
braces++
}
case 40:
case 41:
// line 151 "scan_tokens.rl"
te = p + 1
@ -3285,7 +3295,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
braces--
}
}
case 41:
case 42:
// line 163 "scan_tokens.rl"
te = p + 1
@ -3308,7 +3318,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
case 42:
case 43:
// line 67 "scan_tokens.rl"
te = p + 1
@ -3318,11 +3328,11 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
stack = append(stack, 0)
stack[top] = cs
top++
cs = 989
cs = 990
goto _again
}
}
case 43:
case 44:
// line 77 "scan_tokens.rl"
te = p + 1
@ -3349,31 +3359,31 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
stack = append(stack, 0)
stack[top] = cs
top++
cs = 1003
cs = 1004
goto _again
}
}
case 44:
// line 233 "scan_tokens.rl"
te = p + 1
{
token(TokenInvalid)
}
case 45:
// line 234 "scan_tokens.rl"
te = p + 1
{
token(TokenBadUTF8)
token(TokenInvalid)
}
case 46:
// line 235 "scan_tokens.rl"
te = p + 1
{
token(TokenBadUTF8)
}
case 47:
// line 211 "scan_tokens.rl"
te = p
p--
case 47:
case 48:
// line 212 "scan_tokens.rl"
te = p
@ -3381,7 +3391,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
{
token(TokenNumberLit)
}
case 48:
case 49:
// line 213 "scan_tokens.rl"
te = p
@ -3389,7 +3399,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
{
token(TokenIdent)
}
case 49:
case 50:
// line 215 "scan_tokens.rl"
te = p
@ -3397,59 +3407,67 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
{
token(TokenComment)
}
case 50:
// line 222 "scan_tokens.rl"
case 51:
// line 223 "scan_tokens.rl"
te = p
p--
{
selfToken()
}
case 51:
// line 232 "scan_tokens.rl"
case 52:
// line 233 "scan_tokens.rl"
te = p
p--
{
token(TokenTabs)
}
case 52:
case 53:
// line 234 "scan_tokens.rl"
te = p
p--
{
token(TokenInvalid)
}
case 54:
// line 235 "scan_tokens.rl"
te = p
p--
{
token(TokenBadUTF8)
}
case 53:
case 55:
// line 212 "scan_tokens.rl"
p = (te) - 1
{
token(TokenNumberLit)
}
case 54:
case 56:
// line 213 "scan_tokens.rl"
p = (te) - 1
{
token(TokenIdent)
}
case 55:
// line 222 "scan_tokens.rl"
case 57:
// line 223 "scan_tokens.rl"
p = (te) - 1
{
selfToken()
}
case 56:
// line 234 "scan_tokens.rl"
case 58:
// line 235 "scan_tokens.rl"
p = (te) - 1
{
token(TokenBadUTF8)
}
case 57:
case 59:
// line 1 "NONE"
switch act {
@ -3463,24 +3481,24 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
p = (te) - 1
token(TokenComment)
}
case 21:
case 22:
{
p = (te) - 1
selfToken()
}
case 28:
case 29:
{
p = (te) - 1
token(TokenInvalid)
}
case 29:
case 30:
{
p = (te) - 1
token(TokenBadUTF8)
}
}
// line 3388 "scan_tokens.go"
// line 3402 "scan_tokens.go"
}
}
@ -3501,7 +3519,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
act = 0
// line 3408 "scan_tokens.go"
// line 3422 "scan_tokens.go"
}
}
@ -3527,7 +3545,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
}
}
// line 295 "scan_tokens.rl"
// line 296 "scan_tokens.rl"
// If we fall out here without being in a final state then we've
// encountered something that the scanner can't match, which we'll

1
zcl/zclsyntax/scan_tokens.rl
View File

@ -213,6 +213,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
Ident => { token(TokenIdent) };
Comment => { token(TokenComment) };
Newline => { token(TokenNewline) };
NotEqual => { token(TokenNotEqual); };
GreaterThanEqual => { token(TokenGreaterThanEq); };

82
zcl/zclsyntax/scan_tokens_test.go
View File

@ -40,6 +40,35 @@ func TestScanTokens(t *testing.T) {
},
},
},
{
"\n\n",
[]Token{
{
Type: TokenNewline,
Bytes: []byte("\n"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 1, Line: 2, Column: 1},
},
},
{
Type: TokenNewline,
Bytes: []byte("\n"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 1, Line: 2, Column: 1},
End: zcl.Pos{Byte: 2, Line: 3, Column: 1},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 2, Line: 3, Column: 1},
End: zcl.Pos{Byte: 2, Line: 3, Column: 1},
},
},
},
},
// TokenNumberLit
{
@ -1020,6 +1049,59 @@ EOF
},
},
},
{
"\na = 1\n",
[]Token{
{
Type: TokenNewline,
Bytes: []byte("\n"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
End: zcl.Pos{Byte: 1, Line: 2, Column: 1},
},
},
{
Type: TokenIdent,
Bytes: []byte("a"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 1, Line: 2, Column: 1},
End: zcl.Pos{Byte: 2, Line: 2, Column: 2},
},
},
{
Type: TokenEqual,
Bytes: []byte("="),
Range: zcl.Range{
Start: zcl.Pos{Byte: 3, Line: 2, Column: 3},
End: zcl.Pos{Byte: 4, Line: 2, Column: 4},
},
},
{
Type: TokenNumberLit,
Bytes: []byte("1"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 5, Line: 2, Column: 5},
End: zcl.Pos{Byte: 6, Line: 2, Column: 6},
},
},
{
Type: TokenNewline,
Bytes: []byte("\n"),
Range: zcl.Range{
Start: zcl.Pos{Byte: 6, Line: 2, Column: 6},
End: zcl.Pos{Byte: 7, Line: 3, Column: 1},
},
},
{
Type: TokenEOF,
Bytes: []byte{},
Range: zcl.Range{
Start: zcl.Pos{Byte: 7, Line: 3, Column: 1},
End: zcl.Pos{Byte: 7, Line: 3, Column: 1},
},
},
},
},
// Comments
{