move back to TokenKind

Author: pitaj

compiler/rustc_lexer/src/lib.rs

@@ -104,6 +104,12 @@ pub enum TokenKind {
     /// for emoji identifier recovery, as those are not meant to be ever accepted.
     InvalidPrefix,
 
+    /// Guarded string literal prefix: `#"` or `##`.
+    ///
+    /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
+    /// Split into the component tokens on older editions.
+    GuardedStrPrefix,
+
     /// Examples: `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
     /// suffix, but may be present here on string and float literals. Users of
     /// this type will need to check for and reject that case.
@@ -205,11 +211,6 @@ pub enum LiteralKind {
     ByteStr { terminated: bool },
     /// `c"abc"`, `c"abc`
     CStr { terminated: bool },
-    /// Guarded string literal prefix: `#"` or `##`.
-    ///
-    /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
-    /// Split into the component tokens on older editions.
-    GuardedStrPrefix,
     /// `r"abc"`, `r#"abc"#`, `r####"ab"###"c"####`, `r#"a`. `None` indicates
     /// an invalid literal.
     RawStr { n_hashes: Option<u8> },
@@ -422,8 +423,7 @@ impl Cursor<'_> {
             // Guarded string literal prefix: `#"` or `##`
             '#' if matches!(self.first(), '"' | '#') => {
                 self.bump();
-                let suffix_start = self.pos_within_token();
-                TokenKind::Literal { kind: GuardedStrPrefix, suffix_start }
+                TokenKind::GuardedStrPrefix
             }
 
             // One-symbol tokens.

compiler/rustc_parse/src/lexer/mod.rs

@@ -252,10 +252,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
                     let prefix_span = self.mk_sp(start, lit_start);
                     return (Token::new(self.ident(start), prefix_span), preceded_by_whitespace);
                 }
-                rustc_lexer::TokenKind::Literal {
-                    kind: rustc_lexer::LiteralKind::GuardedStrPrefix,
-                    ..
-                } => self.maybe_report_guarded_str(start, str_before),
+                rustc_lexer::TokenKind::GuardedStrPrefix => self.maybe_report_guarded_str(start, str_before),
                 rustc_lexer::TokenKind::Literal { kind, suffix_start } => {
                     let suffix_start = start + BytePos(suffix_start);
                     let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind);
@@ -607,7 +604,6 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
                 }
                 (kind, self.symbol_from_to(start, end))
             }
-            rustc_lexer::LiteralKind::GuardedStrPrefix => unreachable!(),
         }
     }
 

src/librustdoc/html/highlight.rs

@@ -844,8 +844,8 @@ impl<'src> Classifier<'src> {
                 | LiteralKind::RawCStr { .. } => Class::String,
                 // Number literals.
                 LiteralKind::Float { .. } | LiteralKind::Int { .. } => Class::Number,
-                LiteralKind::GuardedStrPrefix => return no_highlight(sink),
             },
+            TokenKind::GuardedStrPrefix => return no_highlight(sink),
             TokenKind::Ident | TokenKind::RawIdent if lookahead == Some(TokenKind::Bang) => {
                 self.in_macro = true;
                 sink(Highlight::EnterSpan { class: Class::Macro(self.new_span(before, text)) });

src/tools/rust-analyzer/crates/parser/src/lexed_str.rs

@@ -188,7 +188,10 @@ impl<'a> Converter<'a> {
 
                 rustc_lexer::TokenKind::RawIdent => IDENT,
 
-                rustc_lexer::TokenKind::Literal { kind: GuardedStrPrefix, .. } => ERROR,
+                rustc_lexer::TokenKind::GuardedStrPrefix => {
+                    err = "Invalid string literal (reserved syntax)";
+                    ERROR
+                },
 
                 rustc_lexer::TokenKind::Literal { kind, .. } => {
                     self.extend_literal(token_text.len(), kind);