From 29b57b4cc6d61b4652c4a389ab4e4726a2a7faf6 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Wed, 6 Nov 2024 12:48:45 +0900 Subject: [PATCH 1/4] fix trailing new line chars handling (#507) --- decode_test.go | 6 ------ lexer/lexer_test.go | 25 +++++++++++++++++++++++++ scanner/context.go | 38 +++++++++++++++++++++++++------------- 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/decode_test.go b/decode_test.go index 718ca2eb..cac2b81c 100644 --- a/decode_test.go +++ b/decode_test.go @@ -2763,18 +2763,12 @@ func TestDecoder_LiteralWithNewLine(t *testing.T) { { Node: "hello\nworld\n", }, - { - Node: "hello\nworld\n\n", - }, { LastNode: "hello\nworld", }, { LastNode: "hello\nworld\n", }, - { - LastNode: "hello\nworld\n\n", - }, } // struct(want) -> Marshal -> Unmarchal -> struct(got) for _, want := range tests { diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 81b8be9b..810d2d2b 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -2172,6 +2172,31 @@ s: >-3 }, }, }, + { + YAML: ` +| + a + + + +`, + Tokens: token.Tokens{ + { + Type: token.LiteralType, + CharacterType: token.CharacterTypeIndicator, + Indicator: token.BlockScalarIndicator, + Value: "|", + Origin: "\n|\n", + }, + { + Type: token.StringType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "a\n", + Origin: " a\n\n\n\n", + }, + }, + }, } for _, test := range tests { t.Run(test.YAML, func(t *testing.T) { diff --git a/scanner/context.go b/scanner/context.go index 415cf070..5263f3e4 100644 --- a/scanner/context.go +++ b/scanner/context.go @@ -273,27 +273,39 @@ func (c *Context) existsBuffer() bool { func (c *Context) bufferedSrc() []rune { src := c.buf[:c.notSpaceCharPos] - if c.isDocument() && (strings.HasPrefix(c.docOpt, "-") || strings.HasSuffix(c.docOpt, "-")) { - // remove end '\n' character and trailing empty lines + if c.isDocument() { + // remove end '\n' character and trailing empty lines. // https://yaml.org/spec/1.2.2/#8112-block-chomping-indicator - for { - if len(src) > 0 && src[len(src)-1] == '\n' { - src = src[:len(src)-1] - continue + if c.hasTrimAllEndNewlineOpt() { + // If the '-' flag is specified, all trailing newline characters will be removed. + src = []rune(strings.TrimRight(string(src), "\n")) + } else { + // Normally, all but one of the trailing newline characters are removed. + var newLineCharCount int + for i := len(src) - 1; i >= 0; i-- { + if src[i] == '\n' { + newLineCharCount++ + continue + } + break } - break - } - for { - if len(src) > 0 && src[len(src)-1] == ' ' { - src = src[:len(src)-1] - continue + removedNewLineCharCount := newLineCharCount - 1 + for removedNewLineCharCount > 0 { + src = []rune(strings.TrimSuffix(string(src), "\n")) + removedNewLineCharCount-- } - break } + + // If the text ends with a space character, remove all of them. + src = []rune(strings.TrimRight(string(src), " ")) } return src } +func (c *Context) hasTrimAllEndNewlineOpt() bool { + return strings.HasPrefix(c.docOpt, "-") || strings.HasSuffix(c.docOpt, "-") +} + func (c *Context) bufferedToken(pos *token.Position) *token.Token { if c.idx == 0 { return nil From 0faee16367ee5100185888bc93d930a159d53162 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Wed, 6 Nov 2024 16:00:36 +0900 Subject: [PATCH 2/4] fix number parsing (#509) --- lexer/lexer_test.go | 62 ++++++++++++++++++++++- token/token.go | 121 +++++++++++++++++++++++--------------------- 2 files changed, 124 insertions(+), 59 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 810d2d2b..001e886a 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -30,7 +30,7 @@ func TestTokenize(t *testing.T) { YAML: `0_`, Tokens: token.Tokens{ { - Type: token.OctetIntegerType, + Type: token.IntegerType, CharacterType: token.CharacterTypeMiscellaneous, Indicator: token.NotIndicator, Value: "0_", @@ -38,6 +38,54 @@ func TestTokenize(t *testing.T) { }, }, }, + { + YAML: `0x_1A_2B_3C`, + Tokens: token.Tokens{ + { + Type: token.HexIntegerType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "0x_1A_2B_3C", + Origin: "0x_1A_2B_3C", + }, + }, + }, + { + YAML: `+0b1010`, + Tokens: token.Tokens{ + { + Type: token.BinaryIntegerType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "+0b1010", + Origin: "+0b1010", + }, + }, + }, + { + YAML: `0100`, + Tokens: token.Tokens{ + { + Type: token.OctetIntegerType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "0100", + Origin: "0100", + }, + }, + }, + { + YAML: `0o10`, + Tokens: token.Tokens{ + { + Type: token.OctetIntegerType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "0o10", + Origin: "0o10", + }, + }, + }, { YAML: `{} `, @@ -2197,6 +2245,18 @@ s: >-3 }, }, }, + { + YAML: `1x0`, + Tokens: token.Tokens{ + { + Type: token.StringType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "1x0", + Origin: "1x0", + }, + }, + }, } for _, test := range tests { t.Run(test.YAML, func(t *testing.T) { diff --git a/token/token.go b/token/token.go index c2d9a4bc..fce89039 100644 --- a/token/token.go +++ b/token/token.go @@ -523,86 +523,91 @@ type numStat struct { typ numType } -func getNumberStat(str string) *numStat { +func getNumberStat(value string) *numStat { stat := &numStat{} - if str == "" { + if value == "" { return stat } - if str == "-" || str == "." || str == "+" || str == "_" { + dotCount := strings.Count(value, ".") + if dotCount > 1 { return stat } - if str[0] == '_' { + + trimmed := strings.TrimPrefix(strings.TrimPrefix(value, "+"), "-") + + var typ numType + switch { + case strings.HasPrefix(trimmed, "0x"): + trimmed = strings.TrimPrefix(trimmed, "0x") + typ = numTypeHex + case strings.HasPrefix(trimmed, "0o"): + trimmed = strings.TrimPrefix(trimmed, "0o") + typ = numTypeOctet + case strings.HasPrefix(trimmed, "0b"): + trimmed = strings.TrimPrefix(trimmed, "0b") + typ = numTypeBinary + case dotCount == 1: + typ = numTypeFloat + } + + if trimmed == "" { return stat } - dotFound := false - isNegative := false - isExponent := false - if str[0] == '-' { - isNegative = true - } - for idx, c := range str { + + var numCount int + for idx, c := range trimmed { + if isNumber(c) { + numCount++ + continue + } switch c { - case 'x': - if (isNegative && idx == 2) || (!isNegative && idx == 1) { - continue - } - case 'o': - if (isNegative && idx == 2) || (!isNegative && idx == 1) { - continue - } - case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + case '_', '.': continue - case 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F': - if (len(str) > 2 && str[0] == '0' && str[1] == 'x') || - (len(str) > 3 && isNegative && str[1] == '0' && str[2] == 'x') { - // hex number - continue + case 'a', 'b', 'c', 'd', 'f', 'A', 'B', 'C', 'D', 'F': + if typ != numTypeHex && typ != numTypeBinary { + return stat } - if c == 'b' && ((isNegative && idx == 2) || (!isNegative && idx == 1)) { - // binary number + case 'e', 'E': + if typ == numTypeHex || typ == numTypeBinary { continue } - if (c == 'e' || c == 'E') && dotFound { - // exponent - isExponent = true - continue + if typ != numTypeFloat { + return stat } - case '.': - if dotFound { - // multiple dot + + // looks like exponent number. + if len(trimmed) <= idx+2 { return stat } - dotFound = true - continue - case '-': - if idx == 0 || isExponent { - continue + sign := trimmed[idx+1] + if sign != '+' && sign != '-' { + return stat } - case '+': - if idx == 0 || isExponent { - continue + for _, c := range trimmed[idx+2:] { + if !isNumber(c) { + return stat + } } - case '_': - continue + stat.isNum = true + stat.typ = typ + return stat + default: + return stat } - return stat } - stat.isNum = true - switch { - case dotFound: - stat.typ = numTypeFloat - case strings.HasPrefix(str, "0b") || strings.HasPrefix(str, "-0b"): - stat.typ = numTypeBinary - case strings.HasPrefix(str, "0x") || strings.HasPrefix(str, "-0x"): - stat.typ = numTypeHex - case strings.HasPrefix(str, "0o") || strings.HasPrefix(str, "-0o"): - stat.typ = numTypeOctet - case (len(str) > 1 && str[0] == '0') || (len(str) > 1 && str[0] == '-' && str[1] == '0'): - stat.typ = numTypeOctet + if numCount > 1 && trimmed[0] == '0' && typ == numTypeNone { + // YAML 1.1 Spec ? + typ = numTypeOctet } + stat.isNum = true + stat.typ = typ return stat } +func isNumber(c rune) bool { + return c >= '0' && c <= '9' +} + func looksLikeTimeValue(value string) bool { for i, c := range value { switch c { @@ -672,7 +677,7 @@ func LiteralBlockHeader(value string) string { } } -// New create reserved keyword token or number token and other string token +// New create reserved keyword token or number token and other string token. func New(value string, org string, pos *Position) *Token { fn := reservedKeywordMap[value] if fn != nil { From 46a94fdd600fd92717d3eaeb716213656b8457c1 Mon Sep 17 00:00:00 2001 From: Masaaki Goshima Date: Wed, 6 Nov 2024 17:25:05 +0900 Subject: [PATCH 3/4] fix number parsing (#511) --- ast/ast.go | 112 ++++--------------------------- lexer/lexer_test.go | 48 ++++++++++++++ token/token.go | 158 +++++++++++++++++++++----------------------- 3 files changed, 135 insertions(+), 183 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index 47039ec6..e5341df5 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -341,118 +341,30 @@ func Bool(tk *token.Token) *BoolNode { // Integer create node for integer value func Integer(tk *token.Token) *IntegerNode { - switch tk.Type { - case token.BinaryIntegerType: - // skip two characters because binary token starts with '0b' - parsedNum := parseNumber("0b", tk.Value) - if parsedNum.isNegative { - i, _ := strconv.ParseInt(parsedNum.String(), 2, 64) - return &IntegerNode{ - BaseNode: &BaseNode{}, - Token: tk, - Value: i, - } - } - i, _ := strconv.ParseUint(parsedNum.String(), 2, 64) - return &IntegerNode{ - BaseNode: &BaseNode{}, - Token: tk, - Value: i, - } - case token.OctetIntegerType: - // octet token starts with '0o' or '-0o' or '0' or '-0' - parsedNum := parseNumber("0o", tk.Value) - if parsedNum.isNegative { - i, _ := strconv.ParseInt(parsedNum.String(), 8, 64) - return &IntegerNode{ - BaseNode: &BaseNode{}, - Token: tk, - Value: i, - } - } - i, _ := strconv.ParseUint(parsedNum.String(), 8, 64) - return &IntegerNode{ - BaseNode: &BaseNode{}, - Token: tk, - Value: i, - } - case token.HexIntegerType: - // hex token starts with '0x' or '-0x' - parsedNum := parseNumber("0x", tk.Value) - if parsedNum.isNegative { - i, _ := strconv.ParseInt(parsedNum.String(), 16, 64) - return &IntegerNode{ - BaseNode: &BaseNode{}, - Token: tk, - Value: i, - } - } - i, _ := strconv.ParseUint(parsedNum.String(), 16, 64) - return &IntegerNode{ - BaseNode: &BaseNode{}, - Token: tk, - Value: i, - } - } - parsedNum := parseNumber("", tk.Value) - if parsedNum.isNegative { - i, _ := strconv.ParseInt(parsedNum.String(), 10, 64) - return &IntegerNode{ - BaseNode: &BaseNode{}, - Token: tk, - Value: i, - } + var v any + if num := token.ToNumber(tk.Value); num != nil { + v = num.Value } - i, _ := strconv.ParseUint(parsedNum.String(), 10, 64) return &IntegerNode{ BaseNode: &BaseNode{}, Token: tk, - Value: i, - } -} - -type parsedNumber struct { - isNegative bool - num string -} - -func (n *parsedNumber) String() string { - if n.isNegative { - return "-" + n.num - } - return n.num -} - -func parseNumber(prefix, value string) *parsedNumber { - isNegative := value[0] == '-' - trimmed := strings.TrimPrefix(value, "+") - trimmed = strings.TrimPrefix(trimmed, "-") - trimmed = strings.TrimPrefix(trimmed, prefix) - - num := make([]rune, 0, len(trimmed)) - for _, v := range trimmed { - if v == '_' { - continue - } - num = append(num, v) - } - if len(num) == 0 { - num = append(num, '0') - } - return &parsedNumber{ - isNegative: isNegative, - num: string(num), + Value: v, } } // Float create node for float value func Float(tk *token.Token) *FloatNode { - parsedNum := parseNumber("", tk.Value) - f, _ := strconv.ParseFloat(parsedNum.String(), 64) + var v float64 + if num := token.ToNumber(tk.Value); num != nil && num.Type == token.NumberTypeFloat { + value, ok := num.Value.(float64) + if ok { + v = value + } + } return &FloatNode{ BaseNode: &BaseNode{}, Token: tk, - Value: f, + Value: v, } } diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 001e886a..bd2bd24c 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -86,6 +86,18 @@ func TestTokenize(t *testing.T) { }, }, }, + { + YAML: `0.123e+123`, + Tokens: token.Tokens{ + { + Type: token.FloatType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "0.123e+123", + Origin: "0.123e+123", + }, + }, + }, { YAML: `{} `, @@ -2257,6 +2269,42 @@ s: >-3 }, }, }, + { + YAML: `0b98765`, + Tokens: token.Tokens{ + { + Type: token.StringType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "0b98765", + Origin: "0b98765", + }, + }, + }, + { + YAML: `098765`, + Tokens: token.Tokens{ + { + Type: token.StringType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "098765", + Origin: "098765", + }, + }, + }, + { + YAML: `0o98765`, + Tokens: token.Tokens{ + { + Type: token.StringType, + CharacterType: token.CharacterTypeMiscellaneous, + Indicator: token.NotIndicator, + Value: "0o98765", + Origin: "0o98765", + }, + }, + }, } for _, test := range tests { t.Run(test.YAML, func(t *testing.T) { diff --git a/token/token.go b/token/token.go index fce89039..2b3d99d6 100644 --- a/token/token.go +++ b/token/token.go @@ -2,6 +2,7 @@ package token import ( "fmt" + "strconv" "strings" ) @@ -508,104 +509,95 @@ var ( } ) -type numType int +type NumberType string const ( - numTypeNone numType = iota - numTypeBinary - numTypeOctet - numTypeHex - numTypeFloat + NumberTypeDecimal NumberType = "decimal" + NumberTypeBinary NumberType = "binary" + NumberTypeOctet NumberType = "octet" + NumberTypeHex NumberType = "hex" + NumberTypeFloat NumberType = "float" ) -type numStat struct { - isNum bool - typ numType +type NumberValue struct { + Type NumberType + Value any + Text string } -func getNumberStat(value string) *numStat { - stat := &numStat{} - if value == "" { - return stat +func ToNumber(value string) *NumberValue { + if len(value) == 0 { + return nil + } + if strings.HasPrefix(value, "_") { + return nil } dotCount := strings.Count(value, ".") if dotCount > 1 { - return stat + return nil } - trimmed := strings.TrimPrefix(strings.TrimPrefix(value, "+"), "-") + isNegative := strings.HasPrefix(value, "-") + normalized := strings.ReplaceAll(strings.TrimPrefix(strings.TrimPrefix(value, "+"), "-"), "_", "") - var typ numType + var ( + typ NumberType + base int + ) switch { - case strings.HasPrefix(trimmed, "0x"): - trimmed = strings.TrimPrefix(trimmed, "0x") - typ = numTypeHex - case strings.HasPrefix(trimmed, "0o"): - trimmed = strings.TrimPrefix(trimmed, "0o") - typ = numTypeOctet - case strings.HasPrefix(trimmed, "0b"): - trimmed = strings.TrimPrefix(trimmed, "0b") - typ = numTypeBinary + case strings.HasPrefix(normalized, "0x"): + normalized = strings.TrimPrefix(normalized, "0x") + base = 16 + typ = NumberTypeHex + case strings.HasPrefix(normalized, "0o"): + normalized = strings.TrimPrefix(normalized, "0o") + base = 8 + typ = NumberTypeOctet + case strings.HasPrefix(normalized, "0b"): + normalized = strings.TrimPrefix(normalized, "0b") + base = 2 + typ = NumberTypeBinary + case strings.HasPrefix(normalized, "0") && len(normalized) > 1 && dotCount == 0: + base = 8 + typ = NumberTypeOctet case dotCount == 1: - typ = numTypeFloat + typ = NumberTypeFloat + default: + typ = NumberTypeDecimal + base = 10 } - if trimmed == "" { - return stat + text := normalized + if isNegative { + text = "-" + text } - var numCount int - for idx, c := range trimmed { - if isNumber(c) { - numCount++ - continue + var v any + if typ == NumberTypeFloat { + f, err := strconv.ParseFloat(text, 64) + if err != nil { + return nil } - switch c { - case '_', '.': - continue - case 'a', 'b', 'c', 'd', 'f', 'A', 'B', 'C', 'D', 'F': - if typ != numTypeHex && typ != numTypeBinary { - return stat - } - case 'e', 'E': - if typ == numTypeHex || typ == numTypeBinary { - continue - } - if typ != numTypeFloat { - return stat - } - - // looks like exponent number. - if len(trimmed) <= idx+2 { - return stat - } - sign := trimmed[idx+1] - if sign != '+' && sign != '-' { - return stat - } - for _, c := range trimmed[idx+2:] { - if !isNumber(c) { - return stat - } - } - stat.isNum = true - stat.typ = typ - return stat - default: - return stat + v = f + } else if isNegative { + i, err := strconv.ParseInt(text, base, 64) + if err != nil { + return nil } + v = i + } else { + u, err := strconv.ParseUint(text, base, 64) + if err != nil { + return nil + } + v = u } - if numCount > 1 && trimmed[0] == '0' && typ == numTypeNone { - // YAML 1.1 Spec ? - typ = numTypeOctet - } - stat.isNum = true - stat.typ = typ - return stat -} -func isNumber(c rune) bool { - return c >= '0' && c <= '9' + return &NumberValue{ + Type: typ, + Value: v, + Text: text, + } } func looksLikeTimeValue(value string) bool { @@ -632,7 +624,7 @@ func IsNeedQuoted(value string) bool { if _, exists := reservedEncKeywordMap[value]; exists { return true } - if stat := getNumberStat(value); stat.isNum { + if num := ToNumber(value); num != nil { return true } first := value[0] @@ -683,7 +675,7 @@ func New(value string, org string, pos *Position) *Token { if fn != nil { return fn(value, org, pos) } - if stat := getNumberStat(value); stat.isNum { + if num := ToNumber(value); num != nil { tk := &Token{ Type: IntegerType, CharacterType: CharacterTypeMiscellaneous, @@ -692,14 +684,14 @@ func New(value string, org string, pos *Position) *Token { Origin: org, Position: pos, } - switch stat.typ { - case numTypeFloat: + switch num.Type { + case NumberTypeFloat: tk.Type = FloatType - case numTypeBinary: + case NumberTypeBinary: tk.Type = BinaryIntegerType - case numTypeOctet: + case NumberTypeOctet: tk.Type = OctetIntegerType - case numTypeHex: + case NumberTypeHex: tk.Type = HexIntegerType } return tk From 9ffa1596e7ff29f44e383e2dc9e594259d8b1752 Mon Sep 17 00:00:00 2001 From: Shuhei Kitagawa Date: Wed, 6 Nov 2024 18:21:28 +0900 Subject: [PATCH 4/4] Fix ordered map comment (#510) Co-authored-by: Codey Oxley --- decode.go | 1 + yaml_test.go | 488 ++++++++++++++++----------------------------------- 2 files changed, 154 insertions(+), 335 deletions(-) diff --git a/decode.go b/decode.go index 27171f14..afc005c6 100644 --- a/decode.go +++ b/decode.go @@ -154,6 +154,7 @@ func (d *Decoder) setToMapValue(node ast.Node, m map[string]interface{}) error { } func (d *Decoder) setToOrderedMapValue(node ast.Node, m *MapSlice) error { + d.setPathToCommentMap(node) switch n := node.(type) { case *ast.MappingValueNode: if n.Key.Type() == ast.MergeKeyType { diff --git a/yaml_test.go b/yaml_test.go index c2dd6701..da4461ba 100644 --- a/yaml_test.go +++ b/yaml_test.go @@ -744,85 +744,51 @@ hoge: } func Test_CommentToMapOption(t *testing.T) { - t.Run("line comment", func(t *testing.T) { - yml := ` + type testCase struct { + name string + yml string + options []yaml.DecodeOption + expected []struct { + path string + comments []*yaml.Comment + } + } + + tests := []testCase{ + { + name: "line comment", + yml: ` foo: aaa #foo comment bar: #bar comment bbb: ccc #bbb comment baz: x: 10 #x comment -` - var ( - v interface{} - cm = yaml.CommentMap{} - ) - if err := yaml.UnmarshalWithOptions([]byte(yml), &v, yaml.CommentToMap(cm)); err != nil { - t.Fatal(err) - } - expected := []struct { - path string - comments []*yaml.Comment - }{ - { - path: "$.foo", - comments: []*yaml.Comment{yaml.LineComment("foo comment")}, - }, - { - path: "$.bar", - comments: []*yaml.Comment{yaml.LineComment("bar comment")}, - }, - { - path: "$.bar.bbb", - comments: []*yaml.Comment{yaml.LineComment("bbb comment")}, - }, - { - path: "$.baz.x", - comments: []*yaml.Comment{yaml.LineComment("x comment")}, +`, + expected: []struct { + path string + comments []*yaml.Comment + }{ + {"$.foo", []*yaml.Comment{yaml.LineComment("foo comment")}}, + {"$.bar", []*yaml.Comment{yaml.LineComment("bar comment")}}, + {"$.bar.bbb", []*yaml.Comment{yaml.LineComment("bbb comment")}}, + {"$.baz.x", []*yaml.Comment{yaml.LineComment("x comment")}}, }, - } - for _, exp := range expected { - comments := cm[exp.path] - if comments == nil { - t.Fatalf("failed to get path %s", exp.path) - } - if diff := cmp.Diff(exp.comments, comments); diff != "" { - t.Errorf("(-got, +want)\n%s", diff) - } - } - }) - t.Run("line comment2", func(t *testing.T) { - yml := ` + }, + { + name: "line comment2", + yml: ` foo: - bar: baz # comment` - var ( - v interface{} - cm = yaml.CommentMap{} - ) - if err := yaml.UnmarshalWithOptions([]byte(yml), &v, yaml.CommentToMap(cm)); err != nil { - t.Fatal(err) - } - expected := []struct { - path string - comments []*yaml.Comment - }{ - { - path: "$.foo.bar", - comments: []*yaml.Comment{yaml.LineComment(" comment")}, + bar: baz # comment`, + expected: []struct { + path string + comments []*yaml.Comment + }{ + {"$.foo.bar", []*yaml.Comment{yaml.LineComment(" comment")}}, }, - } - for _, exp := range expected { - comments := cm[exp.path] - if comments == nil { - t.Fatalf("failed to get path %s", exp.path) - } - if diff := cmp.Diff(exp.comments, comments); diff != "" { - t.Errorf("(-got, +want)\n%s", diff) - } - } - }) - - t.Run("single head comment", func(t *testing.T) { - yml := ` + }, + { + name: "single head comment", + yml: ` #foo comment foo: aaa #bar comment @@ -832,47 +798,52 @@ bar: baz: #x comment x: 10 -` - var ( - v interface{} - cm = yaml.CommentMap{} - ) - if err := yaml.UnmarshalWithOptions([]byte(yml), &v, yaml.CommentToMap(cm)); err != nil { - t.Fatal(err) - } - expected := []struct { - path string - comments []*yaml.Comment - }{ - { - path: "$.foo", - comments: []*yaml.Comment{yaml.HeadComment("foo comment")}, - }, - { - path: "$.bar", - comments: []*yaml.Comment{yaml.HeadComment("bar comment")}, - }, - { - path: "$.bar.bbb", - comments: []*yaml.Comment{yaml.HeadComment("bbb comment")}, +`, + expected: []struct { + path string + comments []*yaml.Comment + }{ + {"$.foo", []*yaml.Comment{yaml.HeadComment("foo comment")}}, + {"$.bar", []*yaml.Comment{yaml.HeadComment("bar comment")}}, + {"$.bar.bbb", []*yaml.Comment{yaml.HeadComment("bbb comment")}}, + {"$.baz.x", []*yaml.Comment{yaml.HeadComment("x comment")}}, }, - { - path: "$.baz.x", - comments: []*yaml.Comment{yaml.HeadComment("x comment")}, + }, + { + name: "single head comment ordered map", + yml: ` +#first comment +first: value +#second comment +second: + #third comment + third: value + #forth comment + forth: value +#fifth comment +fifth: + #sixth comment + sixth: value + #seventh comment + seventh: value +`, + expected: []struct { + path string + comments []*yaml.Comment + }{ + {"$.first", []*yaml.Comment{yaml.HeadComment("first comment")}}, + {"$.second", []*yaml.Comment{yaml.HeadComment("second comment")}}, + {"$.second.third", []*yaml.Comment{yaml.HeadComment("third comment")}}, + {"$.second.forth", []*yaml.Comment{yaml.HeadComment("forth comment")}}, + {"$.fifth", []*yaml.Comment{yaml.HeadComment("fifth comment")}}, + {"$.fifth.sixth", []*yaml.Comment{yaml.HeadComment("sixth comment")}}, + {"$.fifth.seventh", []*yaml.Comment{yaml.HeadComment("seventh comment")}}, }, - } - for _, exp := range expected { - comments := cm[exp.path] - if comments == nil { - t.Fatalf("failed to get path %s", exp.path) - } - if diff := cmp.Diff(exp.comments, comments); diff != "" { - t.Errorf("(-got, +want)\n%s", diff) - } - } - }) - t.Run("multiple head comments", func(t *testing.T) { - yml := ` + options: []yaml.DecodeOption{yaml.UseOrderedMap()}, + }, + { + name: "multiple head comments", + yml: ` #foo comment #foo comment2 foo: aaa @@ -886,67 +857,20 @@ baz: #x comment #x comment2 x: 10 -` - var ( - v interface{} - cm = yaml.CommentMap{} - ) - if err := yaml.UnmarshalWithOptions([]byte(yml), &v, yaml.CommentToMap(cm)); err != nil { - t.Fatal(err) - } - expected := []struct { - path string - comments []*yaml.Comment - }{ - { - path: "$.foo", - comments: []*yaml.Comment{ - yaml.HeadComment( - "foo comment", - "foo comment2", - ), - }, - }, - { - path: "$.bar", - comments: []*yaml.Comment{ - yaml.HeadComment( - "bar comment", - "bar comment2", - ), - }, - }, - { - path: "$.bar.bbb", - comments: []*yaml.Comment{ - yaml.HeadComment( - "bbb comment", - "bbb comment2", - ), - }, - }, - { - path: "$.baz.x", - comments: []*yaml.Comment{ - yaml.HeadComment( - "x comment", - "x comment2", - ), - }, +`, + expected: []struct { + path string + comments []*yaml.Comment + }{ + {"$.foo", []*yaml.Comment{yaml.HeadComment("foo comment", "foo comment2")}}, + {"$.bar", []*yaml.Comment{yaml.HeadComment("bar comment", "bar comment2")}}, + {"$.bar.bbb", []*yaml.Comment{yaml.HeadComment("bbb comment", "bbb comment2")}}, + {"$.baz.x", []*yaml.Comment{yaml.HeadComment("x comment", "x comment2")}}, }, - } - for _, exp := range expected { - comments := cm[exp.path] - if comments == nil { - t.Fatalf("failed to get path %s", exp.path) - } - if diff := cmp.Diff(exp.comments, comments); diff != "" { - t.Errorf("(-got, +want)\n%s", diff) - } - } - }) - t.Run("foot comment", func(t *testing.T) { - yml := ` + }, + { + name: "foot comment", + yml: ` bar: bbb: ccc #ccc: ddd @@ -956,43 +880,19 @@ baz: #- 3 # foot comment #foot comment2 -` - var ( - v interface{} - cm = yaml.CommentMap{} - ) - if err := yaml.UnmarshalWithOptions([]byte(yml), &v, yaml.CommentToMap(cm)); err != nil { - t.Fatal(err) - } - expected := []struct { - path string - comments []*yaml.Comment - }{ - { - path: "$.bar.bbb", - comments: []*yaml.Comment{yaml.FootComment("ccc: ddd")}, - }, - { - path: "$.baz[1]", - comments: []*yaml.Comment{yaml.FootComment("- 3")}, - }, - { - path: "$.baz", - comments: []*yaml.Comment{yaml.FootComment(" foot comment", "foot comment2")}, +`, + expected: []struct { + path string + comments []*yaml.Comment + }{ + {"$.bar.bbb", []*yaml.Comment{yaml.FootComment("ccc: ddd")}}, + {"$.baz[1]", []*yaml.Comment{yaml.FootComment("- 3")}}, + {"$.baz", []*yaml.Comment{yaml.FootComment(" foot comment", "foot comment2")}}, }, - } - for _, exp := range expected { - comments := cm[exp.path] - if comments == nil { - t.Fatalf("failed to get path %s", exp.path) - } - if diff := cmp.Diff(exp.comments, comments); diff != "" { - t.Errorf("(-got, +want)\n%s", diff) - } - } - }) - t.Run("combination", func(t *testing.T) { - yml := ` + }, + { + name: "combination", + yml: ` # foo head comment # foo head comment2 foo: # foo line comment @@ -1028,138 +928,56 @@ hoge: moga: true # moga line comment # moga foot comment # hoge foot comment -` - var ( - v interface{} - cm = yaml.CommentMap{} - ) - if err := yaml.UnmarshalWithOptions([]byte(yml), &v, yaml.CommentToMap(cm)); err != nil { - t.Fatal(err) - } - expected := []struct { - path string - comments []*yaml.Comment - }{ - { - path: "$.foo", - comments: []*yaml.Comment{ - yaml.HeadComment(" foo head comment", " foo head comment2"), - yaml.LineComment(" foo line comment"), - }, - }, - { - path: "$.foo.a", - comments: []*yaml.Comment{ - yaml.HeadComment(" a head comment"), - yaml.LineComment(" a line comment"), - }, - }, - { - path: "$.foo.a.b", - comments: []*yaml.Comment{ - yaml.HeadComment(" b head comment"), - yaml.LineComment(" b line comment"), - }, - }, - { - path: "$.foo.a.b.c", - comments: []*yaml.Comment{ - yaml.LineComment(" c line comment"), - }, - }, - { - path: "$.o", - comments: []*yaml.Comment{ - yaml.LineComment(" o line comment"), - }, - }, - { - path: "$.o.p", - comments: []*yaml.Comment{ - yaml.HeadComment(" p head comment", " p head comment2"), - yaml.LineComment(" p line comment"), - }, - }, - { - path: "$.o.p.q", - comments: []*yaml.Comment{ - yaml.HeadComment(" q head comment", " q head comment2"), - yaml.LineComment(" q line comment"), - }, - }, - { - path: "$.o.p.q.r", - comments: []*yaml.Comment{ - yaml.LineComment(" r line comment"), - }, - }, - { - path: "$.t.u", - comments: []*yaml.Comment{ - yaml.LineComment(" u line comment"), - }, - }, - { - path: "$.bar", - comments: []*yaml.Comment{ - yaml.HeadComment(" bar head comment"), - yaml.LineComment(" bar line comment"), - }, - }, - { - path: "$.bar.bbb", - comments: []*yaml.Comment{ - yaml.HeadComment(" bbb head comment"), - yaml.LineComment(" bbb line comment"), - yaml.FootComment(" bbb foot comment"), - }, - }, - { - path: "$.baz[0]", - comments: []*yaml.Comment{ - yaml.HeadComment(" sequence head comment"), - yaml.LineComment(" sequence line comment"), - }, - }, - { - path: "$.baz[1]", - comments: []*yaml.Comment{ - yaml.HeadComment(" sequence head comment2"), - yaml.LineComment(" sequence line comment2"), - yaml.FootComment(" sequence foot comment"), - }, - }, - { - path: "$.baz", - comments: []*yaml.Comment{ - yaml.HeadComment(" baz head comment", " baz head comment2"), - yaml.LineComment(" baz line comment"), - }, - }, - { - path: "$.hoge", - comments: []*yaml.Comment{ - yaml.FootComment(" hoge foot comment"), - }, - }, - { - path: "$.hoge.moga", - comments: []*yaml.Comment{ - yaml.LineComment(" moga line comment"), - yaml.FootComment(" moga foot comment"), - }, +`, + expected: []struct { + path string + comments []*yaml.Comment + }{ + {"$.foo", []*yaml.Comment{yaml.HeadComment(" foo head comment", " foo head comment2"), yaml.LineComment(" foo line comment")}}, + {"$.foo.a", []*yaml.Comment{yaml.HeadComment(" a head comment"), yaml.LineComment(" a line comment")}}, + {"$.foo.a.b", []*yaml.Comment{yaml.HeadComment(" b head comment"), yaml.LineComment(" b line comment")}}, + {"$.foo.a.b.c", []*yaml.Comment{yaml.LineComment(" c line comment")}}, + {"$.o", []*yaml.Comment{yaml.LineComment(" o line comment")}}, + {"$.o.p", []*yaml.Comment{yaml.HeadComment(" p head comment", " p head comment2"), yaml.LineComment(" p line comment")}}, + {"$.o.p.q", []*yaml.Comment{yaml.HeadComment(" q head comment", " q head comment2"), yaml.LineComment(" q line comment")}}, + {"$.o.p.q.r", []*yaml.Comment{yaml.LineComment(" r line comment")}}, + {"$.t.u", []*yaml.Comment{yaml.LineComment(" u line comment")}}, + {"$.bar", []*yaml.Comment{yaml.HeadComment(" bar head comment"), yaml.LineComment(" bar line comment")}}, + {"$.bar.bbb", []*yaml.Comment{yaml.HeadComment(" bbb head comment"), yaml.LineComment(" bbb line comment"), yaml.FootComment(" bbb foot comment")}}, + {"$.baz[0]", []*yaml.Comment{yaml.HeadComment(" sequence head comment"), yaml.LineComment(" sequence line comment")}}, + {"$.baz[1]", []*yaml.Comment{yaml.HeadComment(" sequence head comment2"), yaml.LineComment(" sequence line comment2"), yaml.FootComment(" sequence foot comment")}}, + {"$.baz", []*yaml.Comment{yaml.HeadComment(" baz head comment", " baz head comment2"), yaml.LineComment(" baz line comment")}}, + {"$.hoge", []*yaml.Comment{yaml.FootComment(" hoge foot comment")}}, + {"$.hoge.moga", []*yaml.Comment{yaml.LineComment(" moga line comment"), yaml.FootComment(" moga foot comment")}}, }, - } - for _, exp := range expected { - comments := cm[exp.path] - if comments == nil { - t.Fatalf("failed to get path %s", exp.path) + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cm := yaml.CommentMap{} + opts := []yaml.DecodeOption{yaml.CommentToMap(cm)} + opts = append(opts, tc.options...) + + var v interface{} + if err := yaml.UnmarshalWithOptions([]byte(tc.yml), &v, opts...); err != nil { + t.Fatal(err) } - if diff := cmp.Diff(exp.comments, comments); diff != "" { - t.Errorf("%s: (-got, +want)\n%s", exp.path, diff) + + if len(cm) != len(tc.expected) { + t.Fatalf("comment size does not match: got: %d, expected: %d", len(cm), len(tc.expected)) } - } - }) + for _, exp := range tc.expected { + comments := cm[exp.path] + if comments == nil { + t.Fatalf("failed to get path %s", exp.path) + } + if diff := cmp.Diff(exp.comments, comments); diff != "" { + t.Errorf("(-got, +want)\n%s", diff) + } + } + }) + } } func TestCommentMapRoundTrip(t *testing.T) {