zclwrite: absorb lead comments into attributes

The native parser's ranges don't include any surrounding comments, so we
need to do a little more work to pick them out of the surrounding token
sequences.

This just takes care of _lead_ comments, which are those that appear as
whole line comments above the item in question. Line comments, which
appear after the item on the same line, will follow in a later commit.

zcl/zclsyntax/scan_tokens.go

@@ -26,11 +26,11 @@ var _zcltok_actions []byte = []byte{
 	1, 58, 1, 59, 1, 60, 1, 61,
 	1, 62, 1, 63, 1, 64, 1, 65,
 	1, 66, 1, 67, 1, 68, 1, 69,
-	1, 70, 1, 71, 1, 72, 2, 0,
+	1, 70, 1, 71, 1, 72, 1, 73,
-	1, 2, 3, 16, 2, 3, 17, 2,
+	2, 0, 1, 2, 3, 16, 2, 3,
-	3, 28, 2, 3, 29, 2, 3, 40,
+	17, 2, 3, 28, 2, 3, 29, 2,
-	2, 3, 41, 2, 3, 42, 2, 3,
+	3, 40, 2, 3, 41, 2, 3, 42,
-	43, 2, 3, 44,
+	2, 3, 43, 2, 3, 44,
 }
 
 var _zcltok_key_offsets []int16 = []int16{
@@ -184,7 +184,7 @@ var _zcltok_trans_keys []byte = []byte{
 	233, 234, 237, 239, 240, 243, 48, 57,
 	65, 90, 97, 122, 196, 218, 229, 236,
 	10, 170, 181, 183, 186, 128, 150, 152,
-	182, 184, 255, 192, 255, 128, 255, 173,
+	182, 184, 255, 192, 255, 0, 127, 173,
 	130, 133, 146, 159, 165, 171, 175, 255,
 	181, 190, 184, 185, 192, 255, 140, 134,
 	138, 142, 161, 163, 255, 182, 130, 136,
@@ -1410,7 +1410,7 @@ var _zcltok_indicies []int16 = []int16{
 	61, 62, 37, 39, 63, 41, 64, 65,
 	66, 11, 11, 11, 14, 38, 9, 44,
 	9, 11, 11, 11, 11, 9, 11, 11,
-	11, 9, 11, 9, 11, 11, 9, 9,
+	11, 9, 11, 9, 11, 9, 11, 9,
 	9, 9, 9, 9, 11, 9, 9, 9,
 	9, 11, 11, 11, 11, 11, 9, 9,
 	11, 9, 9, 11, 9, 11, 9, 9,
@@ -2063,7 +2063,7 @@ var _zcltok_indicies []int16 = []int16{
 }
 
 var _zcltok_trans_targs []int16 = []int16{
-	947, 1, 952, 2, 3, 955, 947, 5,
+	947, 1, 947, 2, 3, 955, 947, 5,
 	956, 947, 7, 8, 285, 11, 12, 13,
 	14, 15, 286, 287, 18, 288, 20, 21,
 	289, 290, 291, 292, 293, 294, 295, 296,
@@ -2202,12 +2202,12 @@ var _zcltok_trans_targs []int16 = []int16{
 }
 
 var _zcltok_trans_actions []byte = []byte{
-	125, 0, 145, 0, 0, 145, 117, 0,
+	127, 0, 71, 0, 0, 147, 119, 0,
-	5, 121, 0, 0, 0, 0, 0, 0,
+	5, 123, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 95, 0, 0, 0,
+	0, 0, 0, 0, 97, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
@@ -2254,7 +2254,7 @@ var _zcltok_trans_actions []byte = []byte{
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 119, 142, 0, 0, 0, 0,
+	0, 0, 121, 144, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
@@ -2285,7 +2285,7 @@ var _zcltok_trans_actions []byte = []byte{
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 123, 99, 0, 0,
+	0, 0, 0, 0, 125, 101, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
@@ -2298,21 +2298,21 @@ var _zcltok_trans_actions []byte = []byte{
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 27, 5, 0,
-	0, 29, 0, 49, 35, 47, 130, 0,
+	0, 29, 0, 49, 35, 47, 132, 0,
-	0, 0, 0, 69, 55, 67, 136, 0,
+	0, 0, 0, 69, 55, 67, 138, 0,
-	0, 0, 0, 0, 71, 0, 0, 0,
+	0, 0, 0, 0, 73, 0, 0, 0,
-	93, 154, 0, 85, 148, 5, 0, 0,
+	95, 156, 0, 87, 150, 5, 0, 0,
-	87, 0, 89, 0, 97, 0, 0, 0,
+	89, 0, 91, 0, 99, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 5, 5, 5, 151, 151,
+	0, 0, 0, 5, 5, 5, 153, 153,
-	151, 151, 151, 151, 5, 5, 151, 5,
+	153, 153, 153, 153, 5, 5, 153, 5,
-	111, 115, 101, 109, 75, 81, 107, 103,
+	113, 117, 103, 111, 77, 83, 109, 105,
-	0, 0, 79, 73, 77, 105, 0, 0,
+	0, 0, 81, 75, 79, 107, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 83, 91, 113, 0, 0, 0, 0,
+	0, 85, 93, 115, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
@@ -2334,9 +2334,9 @@ var _zcltok_trans_actions []byte = []byte{
 	15, 0, 5, 5, 21, 0, 0, 0,
 	5, 5, 5, 23, 0, 19, 9, 0,
 	17, 7, 25, 0, 0, 0, 0, 37,
-	0, 133, 133, 43, 0, 0, 41, 33,
+	0, 135, 135, 43, 0, 0, 41, 33,
 	39, 31, 45, 0, 0, 0, 57, 0,
-	139, 139, 63, 0, 0, 61, 53, 59,
+	141, 141, 63, 0, 0, 61, 53, 59,
 	51, 65,
 }
 
@@ -2467,8 +2467,8 @@ var _zcltok_to_state_actions []byte = []byte{
 	0, 0, 0, 0, 0, 0, 0, 0,
 	0, 0, 0, 0, 0, 0, 1, 0,
 	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 127, 0, 0, 0,
+	0, 0, 0, 0, 129, 0, 0, 0,
-	0, 0, 0, 127, 0, 0, 0, 0,
+	0, 0, 0, 129, 0, 0, 0, 0,
 	0, 0,
 }
 
@@ -2724,7 +2724,7 @@ var _zcltok_eof_trans []int16 = []int16{
 	766, 766, 766, 770, 770, 772, 774, 772,
 	772, 774, 0, 0, 780, 782, 780, 780,
 	782, 0, 0, 0, 841, 842, 843, 844,
-	1, 844, 844, 847, 848, 844, 844, 844,
+	842, 844, 844, 847, 848, 844, 844, 844,
 	854, 844, 844, 884, 884, 884, 884, 884,
 	884, 884, 884, 884, 884, 884, 884, 884,
 	884, 884, 884, 884, 884, 884, 884, 884,
@@ -3397,62 +3397,69 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 
 				act = 35
 			case 45:
+				// line 242 "scan_tokens.rl"
+
+				te = p + 1
+				{
+					token(TokenComment)
+				}
+			case 46:
 				// line 243 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenNewline)
 				}
-			case 46:
+			case 47:
 				// line 245 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenEqualOp)
 				}
-			case 47:
+			case 48:
 				// line 246 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenNotEqual)
 				}
-			case 48:
+			case 49:
 				// line 247 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenGreaterThanEq)
 				}
-			case 49:
+			case 50:
 				// line 248 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenLessThanEq)
 				}
-			case 50:
+			case 51:
 				// line 249 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenAnd)
 				}
-			case 51:
+			case 52:
 				// line 250 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenOr)
 				}
-			case 52:
+			case 53:
 				// line 251 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					selfToken()
 				}
-			case 53:
+			case 54:
 				// line 154 "scan_tokens.rl"
 
 				te = p + 1
@@ -3460,7 +3467,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 					token(TokenOBrace)
 					braces++
 				}
-			case 54:
+			case 55:
 				// line 159 "scan_tokens.rl"
 
 				te = p + 1
@@ -3481,7 +3488,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 						braces--
 					}
 				}
-			case 55:
+			case 56:
 				// line 171 "scan_tokens.rl"
 
 				te = p + 1
@@ -3511,7 +3518,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 						braces--
 					}
 				}
-			case 56:
+			case 57:
 				// line 71 "scan_tokens.rl"
 
 				te = p + 1
@@ -3525,7 +3532,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 						goto _again
 					}
 				}
-			case 57:
+			case 58:
 				// line 81 "scan_tokens.rl"
 
 				te = p + 1
@@ -3556,27 +3563,27 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 						goto _again
 					}
 				}
-			case 58:
+			case 59:
 				// line 262 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenBadUTF8)
 				}
-			case 59:
+			case 60:
 				// line 263 "scan_tokens.rl"
 
 				te = p + 1
 				{
 					token(TokenInvalid)
 				}
-			case 60:
+			case 61:
 				// line 238 "scan_tokens.rl"
 
 				te = p
 				p--
 
-			case 61:
+			case 62:
 				// line 239 "scan_tokens.rl"
 
 				te = p
@@ -3584,7 +3591,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 				{
 					token(TokenNumberLit)
 				}
-			case 62:
+			case 63:
 				// line 240 "scan_tokens.rl"
 
 				te = p
@@ -3592,7 +3599,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 				{
 					token(TokenIdent)
 				}
-			case 63:
+			case 64:
 				// line 242 "scan_tokens.rl"
 
 				te = p
@@ -3600,7 +3607,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 				{
 					token(TokenComment)
 				}
-			case 64:
+			case 65:
 				// line 251 "scan_tokens.rl"
 
 				te = p
@@ -3608,7 +3615,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 				{
 					selfToken()
 				}
-			case 65:
+			case 66:
 				// line 261 "scan_tokens.rl"
 
 				te = p
@@ -3616,7 +3623,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 				{
 					token(TokenTabs)
 				}
-			case 66:
+			case 67:
 				// line 262 "scan_tokens.rl"
 
 				te = p
@@ -3624,7 +3631,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 				{
 					token(TokenBadUTF8)
 				}
-			case 67:
+			case 68:
 				// line 263 "scan_tokens.rl"
 
 				te = p
@@ -3632,35 +3639,35 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 				{
 					token(TokenInvalid)
 				}
-			case 68:
+			case 69:
 				// line 239 "scan_tokens.rl"
 
 				p = (te) - 1
 				{
 					token(TokenNumberLit)
 				}
-			case 69:
+			case 70:
 				// line 240 "scan_tokens.rl"
 
 				p = (te) - 1
 				{
 					token(TokenIdent)
 				}
-			case 70:
+			case 71:
 				// line 251 "scan_tokens.rl"
 
 				p = (te) - 1
 				{
 					selfToken()
 				}
-			case 71:
+			case 72:
 				// line 262 "scan_tokens.rl"
 
 				p = (te) - 1
 				{
 					token(TokenBadUTF8)
 				}
-			case 72:
+			case 73:
 				// line 1 "NONE"
 
 				switch act {
@@ -3691,7 +3698,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 					}
 				}
 
-				// line 3573 "scan_tokens.go"
+				// line 3578 "scan_tokens.go"
 			}
 		}
 
@@ -3712,7 +3719,7 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
 
 				act = 0
 
-				// line 3593 "scan_tokens.go"
+				// line 3598 "scan_tokens.go"
 			}
 		}
 

zcl/zclsyntax/scan_tokens.rl

@@ -53,8 +53,8 @@ func scanTokens(data []byte, filename string, start zcl.Pos, mode scanMode) []To
         BeginHeredocTmpl = '<<' ('-')? Ident Newline;
 
         Comment = (
-            ("#" any* EndOfLine) |
+            ("#" (any - EndOfLine)* EndOfLine) |
-            ("//" any* EndOfLine) |
+            ("//" (any - EndOfLine)* EndOfLine) |
             ("/*" any* "*/")
         );
 

zclwrite/ast.go

@@ -63,12 +63,16 @@ func (n *Body) Tokens() *TokenSeq {
 }
 
 func (n *Body) AppendItem(node Node) {
+	n.Items = append(n.Items, node)
+	n.AppendUnstructuredTokens(node.Tokens())
+}
+
+func (n *Body) AppendUnstructuredTokens(seq *TokenSeq) {
 	if n.AllTokens == nil {
 		new := make(TokenSeq, 0, 1)
 		n.AllTokens = &new
 	}
-	n.Items = append(n.Items, node)
+	*(n.AllTokens) = append(*(n.AllTokens), seq)
-	*(n.AllTokens) = append(*(n.AllTokens), node.Tokens())
 }
 
 type Attribute struct {
@@ -107,21 +111,10 @@ func (n *Block) walkChildNodes(w internalWalkFunc) {
 	w(n.Body)
 }
 
-// Unstructured represents consecutive sets of tokens within a Body that
+func (n *Block) Tokens() *TokenSeq {
-// aren't part of any particular construct. This includes blank lines
-// and comments that aren't immediately before an attribute or nested block.
-type Unstructured struct {
-	AllTokens *TokenSeq
-}
-
-func (n *Unstructured) Tokens() *TokenSeq {
 	return n.AllTokens
 }
 
-func (n *Unstructured) walkChildNodes(w internalWalkFunc) {
-	// no child nodes
-}
-
 type Expression struct {
 	AllTokens *TokenSeq
 	VarRefs   []*VarRef

zclwrite/parser.go

@@ -76,6 +76,39 @@ func (it inputTokens) Partition(rng zcl.Range) (before, within, after inputToken
 	return
 }
 
+// PartitionIncludeComments is like Partition except the returned "within"
+// range includes any lead and line comments associated with the range.
+func (it inputTokens) PartitionIncludingComments(rng zcl.Range) (before, within, after inputTokens) {
+	start, end := partitionTokens(it.nativeTokens, rng)
+	start = partitionLeadCommentTokens(it.nativeTokens[:start])
+
+	// TODO: Also adjust "end" to include any trailing line comments and the
+	// associated newline.
+
+	before = it.Slice(0, start)
+	within = it.Slice(start, end)
+	after = it.Slice(end, len(it.nativeTokens))
+	return
+
+}
+
+// PartitionWithComments is similar to PartitionIncludeComments but it returns
+// the comments as separate token sequences so that they can be captured into
+// AST attributes.
+func (it inputTokens) PartitionWithComments(rng zcl.Range) (before, leadComments, within, lineComments, after inputTokens) {
+	before, within, after = it.Partition(rng)
+	before, leadComments = before.PartitionLeadComments()
+	lineComments = after.Slice(0, 0) // FIXME: implement this
+	return
+}
+
+func (it inputTokens) PartitionLeadComments() (before, within inputTokens) {
+	start := partitionLeadCommentTokens(it.nativeTokens)
+	before = it.Slice(0, start)
+	within = it.Slice(start, len(it.nativeTokens))
+	return
+}
+
 func (it inputTokens) Slice(start, end int) inputTokens {
 	// When we slice, we create a new slice with no additional capacity because
 	// we expect that these slices will be mutated in order to insert
@@ -96,11 +129,19 @@ func (it inputTokens) Seq() *TokenSeq {
 	return &TokenSeq{it.writerTokens}
 }
 
+func (it inputTokens) Types() []zclsyntax.TokenType {
+	ret := make([]zclsyntax.TokenType, len(it.nativeTokens))
+	for i, tok := range it.nativeTokens {
+		ret[i] = tok.Type
+	}
+	return ret
+}
+
 // parseBody locates the given body within the given input tokens and returns
 // the resulting *Body object as well as the tokens that appeared before and
 // after it.
 func parseBody(nativeBody *zclsyntax.Body, from inputTokens) (inputTokens, *Body, inputTokens) {
-	before, within, after := from.Partition(nativeBody.SrcRange)
+	before, within, after := from.PartitionIncludingComments(nativeBody.SrcRange)
 
 	// The main AST doesn't retain the original source ordering of the
 	// body items, so we need to reconstruct that ordering by inspecting
@@ -123,9 +164,7 @@ func parseBody(nativeBody *zclsyntax.Body, from inputTokens) (inputTokens, *Body
 		beforeItem, item, afterItem := parseBodyItem(nativeItem, remain)
 
 		if beforeItem.Len() > 0 {
-			body.AppendItem(&Unstructured{
+			body.AppendUnstructuredTokens(beforeItem.Seq())
-				AllTokens: beforeItem.Seq(),
-			})
 		}
 		body.AppendItem(item)
 
@@ -133,29 +172,25 @@ func parseBody(nativeBody *zclsyntax.Body, from inputTokens) (inputTokens, *Body
 	}
 
 	if remain.Len() > 0 {
-		body.AppendItem(&Unstructured{
+		body.AppendUnstructuredTokens(remain.Seq())
-			AllTokens: remain.Seq(),
-		})
 	}
 
 	return before, body, after
 }
 
 func parseBodyItem(nativeItem zclsyntax.Node, from inputTokens) (inputTokens, Node, inputTokens) {
-	before, within, after := from.Partition(nativeItem.Range())
+	before, leadComments, within, lineComments, after := from.PartitionWithComments(nativeItem.Range())
 
 	var item Node
 
 	switch tItem := nativeItem.(type) {
 	case *zclsyntax.Attribute:
-		item = parseAttribute(tItem, within)
+		item = parseAttribute(tItem, within, leadComments, lineComments)
 		// TODO: Grab the newline and any line comment from "after" and
 		// write them into the attribute object.
 	case *zclsyntax.Block:
-		// TODO: actually deconstruct the block parts
+		// TODO: implement this
-		item = &Unstructured{
+		panic("block parsing not yet implemented")
-			AllTokens: within.Seq(),
-		}
 	default:
 		// should never happen if caller is behaving
 		panic("unsupported native item type")
@@ -164,10 +199,15 @@ func parseBodyItem(nativeItem zclsyntax.Node, from inputTokens) (inputTokens, No
 	return before, item, after
 }
 
-func parseAttribute(nativeAttr *zclsyntax.Attribute, from inputTokens) *Attribute {
+func parseAttribute(nativeAttr *zclsyntax.Attribute, from, leadComments, lineComments inputTokens) *Attribute {
 	var allTokens TokenSeq
 	attr := &Attribute{}
 
+	if leadComments.Len() > 0 {
+		attr.LeadCommentTokens = leadComments.Seq()
+		allTokens = append(allTokens, attr.LeadCommentTokens)
+	}
+
 	before, nameTokens, from := from.Partition(nativeAttr.NameRange)
 	if before.Len() > 0 {
 		allTokens = append(allTokens, before.Seq())
@@ -189,13 +229,16 @@ func parseAttribute(nativeAttr *zclsyntax.Attribute, from inputTokens) *Attribut
 	attr.Expr = parseExpression(nativeAttr.Expr, exprTokens)
 	allTokens = append(allTokens, attr.Expr.AllTokens)
 
-	// Collect any stragglers, although we shouldn't generally have any since
+	// Collect any stragglers, such as a trailing newline
-	// the newline and any line comments don't get included in the attribute's
-	// range.
 	if from.Len() > 0 {
 		allTokens = append(allTokens, from.Seq())
 	}
 
+	if lineComments.Len() > 0 {
+		attr.LineCommentTokens = lineComments.Seq()
+		allTokens = append(allTokens, attr.LineCommentTokens)
+	}
+
 	attr.AllTokens = &allTokens
 
 	return attr
@@ -286,12 +329,6 @@ func partitionTokens(toks zclsyntax.Tokens, rng zcl.Range) (start, end int) {
 			start = i
 			break
 		}
-
-		if toks[i].Range.Start.Byte < rng.Start.Byte && toks[i].Range.End.Byte < rng.Start.Byte {
-			// We overshot, it seems.
-			start = i
-			break
-		}
 	}
 
 	for i := start; ; i++ {
@@ -309,6 +346,25 @@ func partitionTokens(toks zclsyntax.Tokens, rng zcl.Range) (start, end int) {
 	return start, end
 }
 
+// partitionLeadCommentTokens takes a sequence of tokens that is assumed
+// to immediately precede a construct that can have lead comment tokens,
+// and returns the index into that sequence where the lead comments begin.
+//
+// Lead comments are defined as whole lines containing only comment tokens
+// with no blank lines between. If no such lines are found, the returned
+// index will be len(toks).
+func partitionLeadCommentTokens(toks zclsyntax.Tokens) int {
+	// single-line comments (which is what we're interested in here)
+	// consume their trailing newline, so we can just walk backwards
+	// until we stop seeing comment tokens.
+	for i := len(toks) - 1; i >= 0; i-- {
+		if toks[i].Type != zclsyntax.TokenComment {
+			return i + 1
+		}
+	}
+	return 0
+}
+
 // lexConfig uses the zclsyntax scanner to get a token stream and then
 // rewrites it into this package's token model.
 //

zclwrite/parser_test.go

@@ -115,6 +115,229 @@ func TestParse(t *testing.T) {
 				},
 			},
 		},
+		{
+			"# aye aye aye\na = 1",
+			&Body{
+				Items: []Node{
+					&Attribute{
+						AllTokens: &TokenSeq{
+							&TokenSeq{
+								Tokens{
+									{
+										Type:         zclsyntax.TokenComment,
+										Bytes:        []byte("# aye aye aye\n"),
+										SpacesBefore: 0,
+									},
+								},
+							},
+							&TokenSeq{
+								Tokens{
+									{
+										Type:         zclsyntax.TokenIdent,
+										Bytes:        []byte(`a`),
+										SpacesBefore: 0,
+									},
+								},
+							},
+							&TokenSeq{
+								Tokens{
+									{
+										Type:         zclsyntax.TokenEqual,
+										Bytes:        []byte(`=`),
+										SpacesBefore: 1,
+									},
+								},
+							},
+							&TokenSeq{
+								Tokens{
+									{
+										Type:         zclsyntax.TokenNumberLit,
+										Bytes:        []byte(`1`),
+										SpacesBefore: 1,
+									},
+								},
+							},
+						},
+						LeadCommentTokens: &TokenSeq{Tokens{
+							{
+								Type:         zclsyntax.TokenComment,
+								Bytes:        []byte("# aye aye aye\n"),
+								SpacesBefore: 0,
+							},
+						}},
+						NameTokens: &TokenSeq{Tokens{
+							{
+								Type:         zclsyntax.TokenIdent,
+								Bytes:        []byte(`a`),
+								SpacesBefore: 0,
+							},
+						}},
+						EqualsTokens: &TokenSeq{Tokens{
+							{
+								Type:         zclsyntax.TokenEqual,
+								Bytes:        []byte(`=`),
+								SpacesBefore: 1,
+							},
+						}},
+						Expr: &Expression{
+							AllTokens: &TokenSeq{Tokens{
+								{
+									Type:         zclsyntax.TokenNumberLit,
+									Bytes:        []byte(`1`),
+									SpacesBefore: 1,
+								},
+							}},
+						},
+					},
+				},
+				AllTokens: &TokenSeq{
+					&TokenSeq{
+						&TokenSeq{
+							Tokens{
+								{
+									Type:         zclsyntax.TokenComment,
+									Bytes:        []byte("# aye aye aye\n"),
+									SpacesBefore: 0,
+								},
+							},
+						},
+						&TokenSeq{
+							Tokens{
+								{
+									Type:         zclsyntax.TokenIdent,
+									Bytes:        []byte(`a`),
+									SpacesBefore: 0,
+								},
+							},
+						},
+						&TokenSeq{
+							Tokens{
+								{
+									Type:         zclsyntax.TokenEqual,
+									Bytes:        []byte(`=`),
+									SpacesBefore: 1,
+								},
+							},
+						},
+						&TokenSeq{
+							Tokens{
+								{
+									Type:         zclsyntax.TokenNumberLit,
+									Bytes:        []byte(`1`),
+									SpacesBefore: 1,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			"# bee bee bee\n\nb = 1", // two newlines separate the comment from the attribute
+			&Body{
+				Items: []Node{
+					&Attribute{
+						AllTokens: &TokenSeq{
+							&TokenSeq{
+								Tokens{
+									{
+										Type:         zclsyntax.TokenIdent,
+										Bytes:        []byte(`b`),
+										SpacesBefore: 0,
+									},
+								},
+							},
+							&TokenSeq{
+								Tokens{
+									{
+										Type:         zclsyntax.TokenEqual,
+										Bytes:        []byte(`=`),
+										SpacesBefore: 1,
+									},
+								},
+							},
+							&TokenSeq{
+								Tokens{
+									{
+										Type:         zclsyntax.TokenNumberLit,
+										Bytes:        []byte(`1`),
+										SpacesBefore: 1,
+									},
+								},
+							},
+						},
+						NameTokens: &TokenSeq{Tokens{
+							{
+								Type:         zclsyntax.TokenIdent,
+								Bytes:        []byte(`b`),
+								SpacesBefore: 0,
+							},
+						}},
+						EqualsTokens: &TokenSeq{Tokens{
+							{
+								Type:         zclsyntax.TokenEqual,
+								Bytes:        []byte(`=`),
+								SpacesBefore: 1,
+							},
+						}},
+						Expr: &Expression{
+							AllTokens: &TokenSeq{Tokens{
+								{
+									Type:         zclsyntax.TokenNumberLit,
+									Bytes:        []byte(`1`),
+									SpacesBefore: 1,
+								},
+							}},
+						},
+					},
+				},
+				AllTokens: &TokenSeq{
+					&TokenSeq{
+						Tokens{
+							{
+								Type:         zclsyntax.TokenComment,
+								Bytes:        []byte("# bee bee bee\n"),
+								SpacesBefore: 0,
+							},
+							{
+								Type:         zclsyntax.TokenNewline,
+								Bytes:        []byte("\n"),
+								SpacesBefore: 0,
+							},
+						},
+					},
+					&TokenSeq{
+						&TokenSeq{
+							Tokens{
+								{
+									Type:         zclsyntax.TokenIdent,
+									Bytes:        []byte(`b`),
+									SpacesBefore: 0,
+								},
+							},
+						},
+						&TokenSeq{
+							Tokens{
+								{
+									Type:         zclsyntax.TokenEqual,
+									Bytes:        []byte(`=`),
+									SpacesBefore: 1,
+								},
+							},
+						},
+						&TokenSeq{
+							Tokens{
+								{
+									Type:         zclsyntax.TokenNumberLit,
+									Bytes:        []byte(`1`),
+									SpacesBefore: 1,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	prettyConfig := &pretty.Config{
@@ -306,6 +529,82 @@ func TestPartitionTokens(t *testing.T) {
 	}
 }
 
+func TestPartitionLeadCommentTokens(t *testing.T) {
+	tests := []struct {
+		tokens    zclsyntax.Tokens
+		wantStart int
+	}{
+		{
+			zclsyntax.Tokens{},
+			0,
+		},
+		{
+			zclsyntax.Tokens{
+				{
+					Type: zclsyntax.TokenComment,
+				},
+			},
+			0,
+		},
+		{
+			zclsyntax.Tokens{
+				{
+					Type: zclsyntax.TokenComment,
+				},
+				{
+					Type: zclsyntax.TokenComment,
+				},
+			},
+			0,
+		},
+		{
+			zclsyntax.Tokens{
+				{
+					Type: zclsyntax.TokenComment,
+				},
+				{
+					Type: zclsyntax.TokenNewline,
+				},
+			},
+			2,
+		},
+		{
+			zclsyntax.Tokens{
+				{
+					Type: zclsyntax.TokenComment,
+				},
+				{
+					Type: zclsyntax.TokenNewline,
+				},
+				{
+					Type: zclsyntax.TokenComment,
+				},
+			},
+			2,
+		},
+	}
+
+	prettyConfig := &pretty.Config{
+		Diffable:          true,
+		IncludeUnexported: true,
+		PrintStringers:    true,
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%02d", i), func(t *testing.T) {
+			gotStart := partitionLeadCommentTokens(test.tokens)
+
+			if gotStart != test.wantStart {
+				t.Errorf(
+					"wrong result\ntokens: %s\ngot:   %d\nwant:  %d",
+					prettyConfig.Sprint(test.tokens),
+					gotStart, test.wantStart,
+				)
+			}
+		})
+	}
+}
+
 func TestLexConfig(t *testing.T) {
 	tests := []struct {
 		input string

zclwrite/round_trip_test.go

@@ -19,10 +19,8 @@ baz    = 1
 # this file is awesome
 
 # tossed salads and scrambled eggs
-block "thing" {
+foobar = 1
-  foobar = 1 # quite stylish
 baz    = 1
-}
 
 # and they all lived happily ever after
 `,