zclsyntax: scanning of numeric literals

zcl/zclsyntax/scan_tokens.go

@@ -11,62 +11,75 @@ import (
 var _zcltok_actions []byte = []byte{
 	0, 1, 0, 1, 1, 1, 2, 1, 3,
 	1, 4, 1, 5, 1, 6, 1, 7,
-	1, 8,
+	1, 8, 1, 9, 1, 10,
 }
 
 var _zcltok_key_offsets []byte = []byte{
-	0, 2, 4, 16, 17, 18, 20, 22,
+	0, 4, 6, 8, 10, 24, 25, 26,
+	31, 33, 35,
 }
 
 var _zcltok_trans_keys []byte = []byte{
-	128, 191, 128, 191, 9, 32, 128, 191,
+	43, 45, 48, 57, 48, 57, 128, 191,
+	128, 191, 9, 32, 48, 57, 128, 191,
 	192, 223, 224, 239, 240, 247, 248, 255,
-	9, 32, 128, 191, 128, 191, 128, 191,
+	9, 32, 46, 69, 101, 48, 57, 128,
+	191, 128, 191, 128, 191,
 }
 
 var _zcltok_single_lengths []byte = []byte{
-	0, 0, 2, 1, 1, 0, 0, 0,
+	2, 0, 0, 0, 2, 1, 1, 3,
+	0, 0, 0,
 }
 
 var _zcltok_range_lengths []byte = []byte{
-	1, 1, 5, 0, 0, 1, 1, 1,
+	1, 1, 1, 1, 6, 0, 0, 1,
+	1, 1, 1,
 }
 
 var _zcltok_index_offsets []byte = []byte{
-	0, 2, 4, 12, 14, 16, 18, 20,
+	0, 4, 6, 8, 10, 19, 21, 23,
+	28, 30, 32,
 }
 
 var _zcltok_trans_targs []byte = []byte{
-	2, 2, 0, 2, 3, 4, 2, 5,
+	1, 1, 7, 4, 7, 4, 4, 4,
-	6, 7, 2, 2, 3, 2, 4, 2,
+	2, 4, 5, 6, 7, 4, 8, 9,
-	2, 2, 0, 2, 1, 2, 2, 2,
+	10, 4, 4, 5, 4, 6, 4, 7,
-	2, 2, 2, 2, 2,
+	0, 0, 7, 4, 4, 4, 2, 4,
+	3, 4, 4, 4, 4, 4, 4, 4,
+	4, 4, 4, 4,
 }
 
 var _zcltok_trans_actions []byte = []byte{
-	7, 17, 0, 17, 0, 0, 9, 0,
+	0, 0, 5, 19, 5, 19, 7, 21,
-	5, 5, 9, 7, 0, 13, 0, 11,
+	0, 21, 0, 0, 5, 9, 0, 5,
-	7, 15, 0, 15, 0, 15, 17, 17,
+	5, 9, 7, 0, 15, 0, 11, 5,
-	13, 11, 15, 15, 15,
+	0, 0, 5, 13, 7, 17, 0, 17,
+	0, 17, 19, 19, 21, 21, 15, 11,
+	13, 17, 17, 17,
 }
 
 var _zcltok_to_state_actions []byte = []byte{
-	0, 0, 1, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 1, 0, 0, 0,
+	0, 0, 0,
 }
 
 var _zcltok_from_state_actions []byte = []byte{
-	0, 0, 3, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 3, 0, 0, 0,
+	0, 0, 0,
 }
 
 var _zcltok_eof_trans []byte = []byte{
-	24, 24, 0, 25, 26, 29, 29, 29,
+	36, 36, 38, 38, 0, 39, 40, 41,
+	44, 44, 44,
 }
 
-const zcltok_start int = 2
+const zcltok_start int = 4
-const zcltok_first_final int = 2
+const zcltok_first_final int = 4
 const zcltok_error int = -1
 
-const zcltok_en_main int = 2
+const zcltok_en_main int = 4
 
 // line 13 "scan_tokens.rl"
 
@@ -77,7 +90,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
 		Pos:      start,
 	}
 
-	// line 47 "scan_tokens.rl"
+	// line 50 "scan_tokens.rl"
 
 	// Ragel state
 	cs := 0         // Current State
@@ -98,7 +111,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
 		f.emitToken(ty, ts, te)
 	}
 
-	// line 109 "scan_tokens.go"
+	// line 121 "scan_tokens.go"
 	{
 		cs = zcltok_start
 		ts = 0
@@ -106,7 +119,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
 		act = 0
 	}
 
-	// line 117 "scan_tokens.go"
+	// line 129 "scan_tokens.go"
 	{
 		var _klen int
 		var _trans int
@@ -128,7 +141,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
 
 				ts = p
 
-				// line 138 "scan_tokens.go"
+				// line 150 "scan_tokens.go"
 			}
 		}
 
@@ -204,49 +217,64 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
 				te = p + 1
 
 			case 3:
-				// line 43 "scan_tokens.rl"
+				// line 46 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenInvalid)
 				}
 			case 4:
-				// line 44 "scan_tokens.rl"
+				// line 47 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenBadUTF8)
 				}
 			case 5:
-				// line 41 "scan_tokens.rl"
+				// line 43 "scan_tokens.rl"
 
 				te = p
 				p--
 
 			case 6:
-				// line 42 "scan_tokens.rl"
+				// line 44 "scan_tokens.rl"
+
+				te = p
+				p--
+				{
+					token(TokenNumberLit)
+				}
+			case 7:
+				// line 45 "scan_tokens.rl"
 
 				te = p
 				p--
 				{
 					token(TokenTabs)
 				}
-			case 7:
+			case 8:
-				// line 44 "scan_tokens.rl"
+				// line 47 "scan_tokens.rl"
 
 				te = p
 				p--
 				{
 					token(TokenBadUTF8)
 				}
-			case 8:
+			case 9:
 				// line 44 "scan_tokens.rl"
 
+				p = (te) - 1
+				{
+					token(TokenNumberLit)
+				}
+			case 10:
+				// line 47 "scan_tokens.rl"
+
 				p = (te) - 1
 				{
 					token(TokenBadUTF8)
 				}
-				// line 245 "scan_tokens.go"
+				// line 268 "scan_tokens.go"
 			}
 		}
 
@@ -262,7 +290,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
 
 				ts = 0
 
-				// line 260 "scan_tokens.go"
+				// line 283 "scan_tokens.go"
 			}
 		}
 
@@ -282,7 +310,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
 
 	}
 
-	// line 71 "scan_tokens.rl"
+	// line 74 "scan_tokens.rl"
 
 	// If we fall out here without being in a final state then we've
 	// encountered something that the scanner can't match, which we'll

zcl/zclsyntax/scan_tokens.rl

@@ -30,6 +30,8 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
         );
         BrokenUTF8 = any - AnyUTF8;
 
+        NumberLit = digit (digit|'.'|('e'|'E') ('+'|'-')? digit)*;
+
         # Tabs are not valid, but we accept them in the scanner and mark them
         # as tokens so that we can produce diagnostics advising the user to
         # use spaces instead.
@@ -38,10 +40,11 @@ func scanTokens(data []byte, filename string, start zcl.Pos) []Token {
         Spaces = ' '+;
 
         main := |*
-            Spaces     => {};
+            Spaces           => {};
-            Tabs       => { token(TokenTabs) };
+            NumberLit        => { token(TokenNumberLit) };
-            AnyUTF8    => { token(TokenInvalid) };
+            Tabs             => { token(TokenTabs) };
-            BrokenUTF8 => { token(TokenBadUTF8) };
+            AnyUTF8          => { token(TokenInvalid) };
+            BrokenUTF8       => { token(TokenBadUTF8) };
         *|;
 
     }%%

zcl/zclsyntax/scan_tokens_test.go

@@ -13,6 +13,7 @@ func TestScanTokens(t *testing.T) {
 		input string
 		want  []Token
 	}{
+		// Empty input
 		{
 			``,
 			[]Token{
@@ -39,6 +40,115 @@ func TestScanTokens(t *testing.T) {
 				},
 			},
 		},
+
+		// TokenNumberLit
+		{
+			`1`,
+			[]Token{
+				{
+					Type:  TokenNumberLit,
+					Bytes: []byte(`1`),
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   zcl.Pos{Byte: 1, Line: 1, Column: 2},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 1, Line: 1, Column: 2},
+						End:   zcl.Pos{Byte: 1, Line: 1, Column: 2},
+					},
+				},
+			},
+		},
+		{
+			`12`,
+			[]Token{
+				{
+					Type:  TokenNumberLit,
+					Bytes: []byte(`12`),
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   zcl.Pos{Byte: 2, Line: 1, Column: 3},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 2, Line: 1, Column: 3},
+						End:   zcl.Pos{Byte: 2, Line: 1, Column: 3},
+					},
+				},
+			},
+		},
+		{
+			`12.3`,
+			[]Token{
+				{
+					Type:  TokenNumberLit,
+					Bytes: []byte(`12.3`),
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   zcl.Pos{Byte: 4, Line: 1, Column: 5},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
+						End:   zcl.Pos{Byte: 4, Line: 1, Column: 5},
+					},
+				},
+			},
+		},
+		{
+			`1e2`,
+			[]Token{
+				{
+					Type:  TokenNumberLit,
+					Bytes: []byte(`1e2`),
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   zcl.Pos{Byte: 3, Line: 1, Column: 4},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 3, Line: 1, Column: 4},
+						End:   zcl.Pos{Byte: 3, Line: 1, Column: 4},
+					},
+				},
+			},
+		},
+		{
+			`1e+2`,
+			[]Token{
+				{
+					Type:  TokenNumberLit,
+					Bytes: []byte(`1e+2`),
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 0, Line: 1, Column: 1},
+						End:   zcl.Pos{Byte: 4, Line: 1, Column: 5},
+					},
+				},
+				{
+					Type:  TokenEOF,
+					Bytes: []byte{},
+					Range: zcl.Range{
+						Start: zcl.Pos{Byte: 4, Line: 1, Column: 5},
+						End:   zcl.Pos{Byte: 4, Line: 1, Column: 5},
+					},
+				},
+			},
+		},
+
+		// Invalid things
 		{
 			`|`,
 			[]Token{