chore: 🤖 basic finish parse with comments

rome · ematipico · Jul 21, 2023 · Apr 17, 2023 · Apr 17, 2023 · Apr 17, 2023
commit 8a2531e46dddde2e37220c2feca5d9399ead5d54
diff --git a/crates/rome_json_parser/src/lexer/mod.rs b/crates/rome_json_parser/src/lexer/mod.rs
@@ -9,6 +9,8 @@ use rome_parser::diagnostic::ParseDiagnostic;
 use std::iter::FusedIterator;
 use std::ops::Add;
 
+use crate::JsonParserConfig;
+
 pub struct Token {
     kind: JsonSyntaxKind,
     range: TextRange,
@@ -34,6 +36,7 @@ pub(crate) struct Lexer<'src> {
     position: usize,
 
     diagnostics: Vec<ParseDiagnostic>,
+    config: JsonParserConfig,
 }
 
 impl<'src> Lexer<'src> {
@@ -43,6 +46,7 @@ impl<'src> Lexer<'src> {
             source: string,
             position: 0,
             diagnostics: vec![],
+            config: JsonParserConfig::default(),
         }
     }
 
@@ -695,10 +699,12 @@ impl<'src> Lexer<'src> {
                         b'*' if self.peek_byte() == Some(b'/') => {
                             self.advance(2);
 
-                            self.diagnostics.push(ParseDiagnostic::new(
-                                "JSON standard does not allow comments.",
-                                start..self.text_position(),
-                            ));
+                            if !self.config.allow_comments {
+                                self.diagnostics.push(ParseDiagnostic::new(
+                                    "JSON standard does not allow comments.",
+                                    start..self.text_position(),
+                                ));
+                            }
 
                             if has_newline {
                                 return MULTILINE_COMMENT;
@@ -739,16 +745,23 @@ impl<'src> Lexer<'src> {
                     }
                 }
 
-                self.diagnostics.push(ParseDiagnostic::new(
-                    "JSON standard does not allow comments.",
-                    start..self.text_position(),
-                ));
+                if !self.config.allow_comments {
+                    self.diagnostics.push(ParseDiagnostic::new(
+                        "JSON standard does not allow comments.",
+                        start..self.text_position(),
+                    ));
+                }
 
                 COMMENT
             }
             _ => self.eat_unexpected_character(),
         }
     }
+
+    pub(crate) fn with_config(mut self, config: JsonParserConfig) -> Self {
+        self.config = config;
+        self
+    }
 }
 
 impl Iterator for Lexer<'_> {

diff --git a/crates/rome_json_parser/src/parser.rs b/crates/rome_json_parser/src/parser.rs
@@ -1,6 +1,6 @@
 use crate::token_source::JsonTokenSource;
 use rome_json_syntax::JsonSyntaxKind;
-use rome_parser::diagnostic::merge_diagnostics;
+use rome_parser::diagnostic::{expected_token, merge_diagnostics};
 use rome_parser::event::Event;
 use rome_parser::prelude::*;
 use rome_parser::token_source::Trivia;
@@ -12,16 +12,16 @@ pub(crate) struct JsonParser<'source> {
     config: JsonParserConfig,
 }
 
-#[derive(Default)]
+#[derive(Default, Debug, Clone, Copy)]
 pub struct JsonParserConfig {
-    allow_comments: bool,
+    pub allow_comments: bool,
 }
 
 impl<'source> JsonParser<'source> {
     pub fn new(source: &'source str, config: JsonParserConfig) -> Self {
         Self {
             context: ParserContext::default(),
-            source: JsonTokenSource::from_str(source),
+            source: JsonTokenSource::from_str(source, config),
             config,
         }
     }
@@ -61,4 +61,27 @@ impl<'source> Parser for JsonParser<'source> {
     fn source_mut(&mut self) -> &mut Self::Source {
         &mut self.source
     }
+
+    /// Try to eat a specific token kind, if the kind is not there then adds an error to the events stack.
+    fn expect(&mut self, kind: Self::Kind) -> bool {
+        if self.eat(kind) {
+            true
+        } else {
+            if self.config.allow_comments {
+                println!("something");
+                while matches!(
+                    kind,
+                    JsonSyntaxKind::COMMENT | JsonSyntaxKind::MULTILINE_COMMENT
+                ) {
+                    self.bump_any();
+                }
+            }
+            if self.eat(kind) {
+                return true;
+            }
+
+            self.error(expected_token(kind));
+            false
+        }
+    }
 }
diff --git a/crates/rome_json_parser/src/token_source.rs b/crates/rome_json_parser/src/token_source.rs
@@ -1,4 +1,5 @@
 use crate::lexer::Lexer;
+use crate::JsonParserConfig;
 use rome_json_syntax::JsonSyntaxKind::{EOF, TOMBSTONE};
 use rome_json_syntax::{JsonSyntaxKind, TextRange};
 use rome_parser::diagnostic::ParseDiagnostic;
@@ -12,18 +13,20 @@ pub(crate) struct JsonTokenSource<'source> {
     current: JsonSyntaxKind,
     current_range: TextRange,
     preceding_line_break: bool,
+    config: JsonParserConfig,
 }
 
 impl<'source> JsonTokenSource<'source> {
-    pub fn from_str(source: &'source str) -> Self {
-        let lexer = Lexer::from_str(source);
+    pub fn from_str(source: &'source str, config: JsonParserConfig) -> Self {
+        let lexer = Lexer::from_str(source).with_config(config);
 
         let mut source = Self {
             lexer,
             trivia: Vec::new(),
             current: TOMBSTONE,
             current_range: TextRange::default(),
             preceding_line_break: false,
+            config,
         };
 
         source.next_non_trivia_token(true);
@@ -39,6 +42,18 @@ impl<'source> JsonTokenSource<'source> {
 
             match trivia_kind {
                 Err(_) => {
+                    if self.config.allow_comments && token.kind().is_comments() {
+                        dbg!(token.kind());
+                        let trivia_kind = match token.kind() {
+                            JsonSyntaxKind::COMMENT => TriviaPieceKind::SingleLineComment,
+                            JsonSyntaxKind::MULTILINE_COMMENT => TriviaPieceKind::MultiLineComment,
+                            _ => unreachable!(),
+                        };
+                        self.trivia
+                            .push(Trivia::new(trivia_kind, token.range(), trailing));
+                        continue;
+                    }
+
                     self.current = token.kind();
                     self.current_range = token.range();
 

diff --git a/crates/rome_json_parser/tests/json_test_suite/allow_comments/basic.json b/crates/rome_json_parser/tests/json_test_suite/allow_comments/basic.json
@@ -0,0 +1 @@
+/*leading*/ [1,true] // trivial
diff --git a/crates/rome_json_parser/tests/json_test_suite/allow_comments/basic.json.snap b/crates/rome_json_parser/tests/json_test_suite/allow_comments/basic.json.snap
@@ -0,0 +1,52 @@
+---
+source: crates/rome_json_parser/tests/spec_test.rs
+expression: snapshot
+---
+
+## Input
+
+```json
+/*leading*/ [1,true] // trivial
+
+```
+
+
+## AST
+
+```
+JsonRoot {
+    value: JsonArrayValue {
+        l_brack_token: [email protected] "[" [Comments("/*leading*/"), Whitespace(" ")] [],
+        elements: JsonArrayElementList [
+            JsonNumberValue {
+                value_token: [email protected] "1" [] [],
+            },
+            [email protected] "," [] [],
+            JsonBooleanValue {
+                value_token: [email protected] "true" [] [],
+            },
+        ],
+        r_brack_token: [email protected] "]" [] [Whitespace(" "), Comments("// trivial")],
+    },
+    eof_token: [email protected] "" [Newline("\n")] [],
+}
+```
+
+## CST
+
+```
+0: [email protected]
+  0: [email protected]
+    0: [email protected] "[" [Comments("/*leading*/"), Whitespace(" ")] []
+    1: [email protected]
+      0: [email protected]
+        0: [email protected] "1" [] []
+      1: [email protected] "," [] []
+      2: [email protected]
+        0: [email protected] "true" [] []
+    2: [email protected] "]" [] [Whitespace(" "), Comments("// trivial")]
+  1: [email protected] "" [Newline("\n")] []
+
+```
+
+
diff --git a/crates/rome_json_parser/tests/spec_test.rs b/crates/rome_json_parser/tests/spec_test.rs
@@ -16,12 +16,13 @@ pub enum ExpectedOutcome {
     Undefined,
 }
 
-pub fn run(test_case: &str, _snapshot_name: &str, test_directory: &str, outcome: &str) {
-    let outcome = match outcome {
+pub fn run(test_case: &str, _snapshot_name: &str, test_directory: &str, outcome_str: &str) {
+    let outcome = match outcome_str {
         "ok" => ExpectedOutcome::Pass,
         "error" => ExpectedOutcome::Fail,
         "undefined" => ExpectedOutcome::Undefined,
-        _ => panic!("Invalid expected outcome {outcome}"),
+        "allow_comments" => ExpectedOutcome::Pass,
+        _ => panic!("Invalid expected outcome {outcome_str}"),
     };
 
     let test_case_path = Path::new(test_case);
@@ -35,7 +36,10 @@ pub fn run(test_case: &str, _snapshot_name: &str, test_directory: &str, outcome:
     let content = fs::read_to_string(test_case_path)
         .expect("Expected test path to be a readable file in UTF8 encoding");
 
-    let parsed = parse_json(&content, JsonParserConfig::default());
+    let parse_conifg = JsonParserConfig {
+        allow_comments: outcome_str == "allow_comments",
+    };
+    let parsed = parse_json(&content, parse_conifg);
     let formatted_ast = format!("{:#?}", parsed.tree());
 
     let mut snapshot = String::new();

diff --git a/crates/rome_json_parser/tests/spec_tests.rs b/crates/rome_json_parser/tests/spec_tests.rs
@@ -16,3 +16,8 @@ mod undefined {
     //! parsers are free to accept or reject content
     tests_macros::gen_tests! {"tests/json_test_suite/undefined/*.json", crate::spec_test::run, "undefined"}
 }
+
+mod allow_comments {
+    //! Tests should pass even with comments in json
+    tests_macros::gen_tests! {"tests/json_test_suite/allow_comments/*.json", crate::spec_test::run, "allow_comments"}
+}
diff --git a/crates/rome_json_syntax/src/lib.rs b/crates/rome_json_syntax/src/lib.rs
@@ -30,6 +30,13 @@ impl JsonSyntaxKind {
         matches!(self, JsonSyntaxKind::NEWLINE | JsonSyntaxKind::WHITESPACE)
     }
 
+    pub fn is_comments(self) -> bool {
+        matches!(
+            self,
+            JsonSyntaxKind::COMMENT | JsonSyntaxKind::MULTILINE_COMMENT
+        )
+    }
+
     #[inline]
     pub const fn is_keyword(self) -> bool {
         matches!(self, T![null] | T![true] | T![false])

diff --git a/crates/rome_service/src/configuration/json.rs b/crates/rome_service/src/configuration/json.rs
@@ -1,9 +1,5 @@
-use std::str::FromStr;
-
 use bpaf::Bpaf;
-use indexmap::IndexSet;
 use serde::{Deserialize, Serialize};
-use serde_json::from_str;
 
 use super::StringSet;
 #[derive(Default, Debug, Deserialize, Serialize, Eq, PartialEq, Clone, Bpaf)]