ISLE: use string references in Token::Symbol

Replace `String` with `&str` in `Token::Symbol`, so we don't need to
heap allocate a new string for every symbol that is lexed.

Copyright (c) 2024, Arm Limited.

Signed-off-by: Karl Meakin <karl.meakin@arm.com>

Author: Kmeakin

cranelift/isle/isle/src/lexer.rs

@@ -14,7 +14,7 @@ type Result<T> = std::result::Result<T, Error>;
 pub struct Lexer<'src> {
     src: &'src str,
     pos: Pos,
-    lookahead: Option<(Pos, Token)>,
+    lookahead: Option<(Pos, Token<'src>)>,
 }
 
 /// A source position.
@@ -46,14 +46,14 @@ impl Pos {
 }
 
 /// A token of ISLE source.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub enum Token {
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum Token<'src> {
     /// Left paren.
     LParen,
     /// Right paren.
     RParen,
     /// A symbol, e.g. `Foo`.
-    Symbol(String),
+    Symbol(&'src str),
     /// An integer.
     Int(i128),
     /// `@`
@@ -88,7 +88,7 @@ impl<'src> Lexer<'src> {
         }
     }
 
-    fn next_token(&mut self) -> Result<Option<(Pos, Token)>> {
+    fn next_token(&mut self) -> Result<Option<(Pos, Token<'src>)>> {
         fn is_sym_first_char(c: u8) -> bool {
             match c {
                 b'-' | b'0'..=b'9' | b'(' | b')' | b';' => false,
@@ -149,7 +149,7 @@ impl<'src> Lexer<'src> {
                 let end = self.pos.offset;
                 let s = &self.src[start..end];
                 debug_assert!(!s.is_empty());
-                Ok(Some((start_pos, Token::Symbol(s.to_string()))))
+                Ok(Some((start_pos, Token::Symbol(s))))
             }
             c @ (b'0'..=b'9' | b'-') => {
                 let start_pos = self.pos();
@@ -225,7 +225,7 @@ impl<'src> Lexer<'src> {
     }
 
     /// Get the next token from this lexer's token stream, if any.
-    pub fn next(&mut self) -> Result<Option<(Pos, Token)>> {
+    pub fn next(&mut self) -> Result<Option<(Pos, Token<'src>)>> {
         let tok = self.lookahead.take();
         self.reload()?;
         Ok(tok)
@@ -239,7 +239,7 @@ impl<'src> Lexer<'src> {
     }
 
     /// Peek ahead at the next token.
-    pub fn peek(&self) -> Option<&(Pos, Token)> {
+    pub fn peek(&self) -> Option<&(Pos, Token<'src>)> {
         self.lookahead.as_ref()
     }
 
@@ -253,7 +253,7 @@ impl<'src> Lexer<'src> {
     }
 }
 
-impl Token {
+impl<'src> Token<'src> {
     /// Is this an `Int` token?
     pub fn is_int(&self) -> bool {
         matches!(self, Token::Int(_))
@@ -285,9 +285,9 @@ mod test {
             lex(";; comment\n; another\r\n   \t(one two three 23 -568  )\n"),
             [
                 Token::LParen,
-                Token::Symbol("one".to_string()),
-                Token::Symbol("two".to_string()),
-                Token::Symbol("three".to_string()),
+                Token::Symbol("one"),
+                Token::Symbol("two"),
+                Token::Symbol("three"),
                 Token::Int(23),
                 Token::Int(-568),
                 Token::RParen
@@ -297,7 +297,7 @@ mod test {
 
     #[test]
     fn ends_with_sym() {
-        assert_eq!(lex("asdf"), [Token::Symbol("asdf".to_string())]);
+        assert_eq!(lex("asdf"), [Token::Symbol("asdf")]);
     }
 
     #[test]
@@ -311,11 +311,11 @@ mod test {
             lex("(+ [] => !! _test!;comment\n)"),
             [
                 Token::LParen,
-                Token::Symbol("+".to_string()),
-                Token::Symbol("[]".to_string()),
-                Token::Symbol("=>".to_string()),
-                Token::Symbol("!!".to_string()),
-                Token::Symbol("_test!".to_string()),
+                Token::Symbol("+"),
+                Token::Symbol("[]"),
+                Token::Symbol("=>"),
+                Token::Symbol("!!"),
+                Token::Symbol("_test!"),
                 Token::RParen,
             ]
         );

cranelift/isle/isle/src/parser.rs

@@ -16,8 +16,8 @@ pub fn parse(lexer: Lexer) -> Result<Vec<Def>> {
 ///
 /// Takes in a lexer and creates an AST.
 #[derive(Clone, Debug)]
-struct Parser<'a> {
-    lexer: Lexer<'a>,
+struct Parser<'src> {
+    lexer: Lexer<'src>,
 }
 
 /// Used during parsing a `(rule ...)` to encapsulate some form that
@@ -28,9 +28,9 @@ enum IfLetOrExpr {
     Expr(Expr),
 }
 
-impl<'a> Parser<'a> {
+impl<'src> Parser<'src> {
     /// Construct a new parser from the given lexer.
-    pub fn new(lexer: Lexer<'a>) -> Parser<'a> {
+    pub fn new(lexer: Lexer<'src>) -> Parser<'src> {
         Parser { lexer }
     }
 
@@ -41,7 +41,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn expect<F: Fn(&Token) -> bool>(&mut self, f: F) -> Result<Token> {
+    fn expect<F: Fn(&Token<'src>) -> bool>(&mut self, f: F) -> Result<Token<'src>> {
         if let Some(&(pos, ref peek)) = self.lexer.peek() {
             if !f(peek) {
                 return Err(self.error(pos, format!("Unexpected token {peek:?}")));
@@ -52,7 +52,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn eat<F: Fn(&Token) -> bool>(&mut self, f: F) -> Result<Option<Token>> {
+    fn eat<F: Fn(&Token<'src>) -> bool>(&mut self, f: F) -> Result<Option<Token<'src>>> {
         if let Some(&(_pos, ref peek)) = self.lexer.peek() {
             if !f(peek) {
                 return Ok(None);
@@ -63,7 +63,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn is<F: Fn(&Token) -> bool>(&self, f: F) -> bool {
+    fn is<F: Fn(&Token<'src>) -> bool>(&self, f: F) -> bool {
         if let Some((_, peek)) = self.lexer.peek() {
             f(peek)
         } else {
@@ -108,10 +108,7 @@ impl<'a> Parser<'a> {
     }
 
     fn is_spec_bool(&self) -> bool {
-        self.is(|tok| match tok {
-            Token::Symbol(tok_s) if tok_s == "$true" || tok_s == "$false" => true,
-            _ => false,
-        })
+        self.is(|tok| matches!(tok, Token::Symbol("$true" | "$false")))
     }
 
     fn expect_lparen(&mut self) -> Result<()> {
@@ -124,7 +121,7 @@ impl<'a> Parser<'a> {
         self.expect(|tok| *tok == Token::At).map(|_| ())
     }
 
-    fn expect_symbol(&mut self) -> Result<String> {
+    fn expect_symbol(&mut self) -> Result<&'src str> {
         match self.expect(Token::is_sym)? {
             Token::Symbol(s) => Ok(s),
             _ => unreachable!(),
@@ -133,7 +130,7 @@ impl<'a> Parser<'a> {
 
     fn eat_sym_str(&mut self, s: &str) -> Result<bool> {
         self.eat(|tok| match tok {
-            Token::Symbol(ref tok_s) if tok_s == s => true,
+            Token::Symbol(ref tok_s) if *tok_s == s => true,
             _ => false,
         })
         .map(|token| token.is_some())
@@ -424,7 +421,7 @@ impl<'a> Parser<'a> {
             }
             if self.is_sym() && !self.is_spec_bit_vector() {
                 let sym = self.expect_symbol()?;
-                if let Ok(op) = self.parse_spec_op(sym.as_str()) {
+                if let Ok(op) = self.parse_spec_op(sym) {
                     let mut args: Vec<SpecExpr> = vec![];
                     while !self.is_rparen() {
                         args.push(self.parse_spec_expr()?);
@@ -531,7 +528,7 @@ impl<'a> Parser<'a> {
     fn parse_spec_bool(&mut self) -> Result<i8> {
         let pos = self.pos();
         let s = self.expect_symbol()?;
-        match s.as_str() {
+        match s {
             "$true" => Ok(1),
             "$false" => Ok(0),
             x => Err(self.error(pos, format!("Not a valid spec boolean: {x}"))),