1use crate::string_block::lex_str_block_test;2use core::ops::Range;3use logos::Logos;4use rowan::{Checkpoint, TextRange, TextSize};5use std::{convert::TryFrom, iter::Peekable};67#[derive(Logos, Debug, PartialEq, Hash, Eq, PartialOrd, Ord, Clone, Copy)]8#[repr(u16)]9pub enum SyntaxKind {10 #[token("assert")]11 KeywordAssert = 0,1213 #[token("else")]14 KeywordElse,1516 #[token("error")]17 KeywordError,1819 #[token("false")]20 KeywordFalse,2122 #[token("for")]23 KeywordFor,2425 #[token("function")]26 KeywordFunction,2728 #[token("if")]29 KeywordIf,3031 #[token("import")]32 KeywordImport,3334 #[token("importstr")]35 KeywordImportStr,3637 #[token("local")]38 KeywordLocal,3940 #[token("null")]41 KeywordNull,4243 #[token("tailstrict")]44 KeywordTailStrict,4546 #[token("then")]47 KeywordThen,4849 #[token("self")]50 KeywordSelf,5152 #[token("super")]53 KeywordSuper,5455 #[token("true")]56 KeywordTrue,5758 #[regex(r"[_a-zA-Z][_a-zA-Z0-9]*")]59 Ident,6061 #[regex(r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")]62 Number,6364 #[regex(r"(?:0|[1-9][0-9]*)\.[^0-9]")]65 ErrorNumJunkAfterDecimalPoint,6667 #[regex(r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][^+\-0-9]")]68 ErrorNumJunkAfterExponent,6970 #[regex(r"(?:0|[1-9][0-9]*)(?:\.[0-9]+)?[eE][+-][^0-9]")]71 ErrorNumJunkAfterExponentSign,7273 #[token("{")]74 SymbolLeftBrace,7576 #[token("}")]77 SymbolRightBrace,7879 #[token("[")]80 SymbolLeftBracket,8182 #[token("]")]83 SymbolRightBracket,8485 #[token(",")]86 SymbolComma,8788 #[token(".")]89 SymbolDot,9091 #[token("(")]92 LParen,9394 #[token(")")]95 RParen,9697 #[token(";")]98 SymbolSemi,99 #[token(":")]100 SymbolColon,101102 #[token("$")]103 SymbolDollar,104105 #[token("*")]106 OpMul,107 #[token("/")]108 OpDiv,109 #[token("%")]110 OpMod,111 #[token("+")]112 OpPlus,113 #[token("-")]114 OpMinus,115 #[token("<<")]116 OpShiftLeft,117 #[token(">>")]118 OpShiftRight,119 #[token("<")]120 OpLessThan,121 #[token(">")]122 OpGreaterThan,123 #[token("<=")]124 OpLessThanOrEqual,125 #[token(">=")]126 OpGreaterThanOrEqual,127 #[token("==")]128 OpEqual,129 #[token("!=")]130 OpNotEqual,131 #[token("&")]132 OpBitAnd,133 #[token("^")]134 OpBitXor,135 #[token("|")]136 OpBitOr,137 #[token("&&")]138 OpAnd,139 #[token("||")]140 OpOr,141 #[token("in")]142 OpIn,143 #[token("!")]144 OpNot,145 #[token("~")]146 OpBitNegate,147 #[token("=")]148 SymbolAssign,149150 #[regex("\"(?s:[^\"\\\\]|\\\\.)*\"")]151 StringDoubleQuoted,152153 #[regex("'(?s:[^'\\\\]|\\\\.)*'")]154 StringSingleQuoted,155156 #[regex("@\"(?:[^\"]|\"\")*\"")]157 StringDoubleVerbatim,158159 #[regex("@'(?:[^']|'')*'")]160 StringSingleVerbatim,161162 #[regex(r"\|\|\|", lex_str_block_test)]163 StringBlock, 164165 #[regex("\"(?s:[^\"\\\\]|\\\\.)*")]166 ErrorStringDoubleQuotedUnterminated,167168 #[regex("'(?s:[^'\\\\]|\\\\.)*")]169 ErrorStringSingleQuotedUnterminated,170171 #[regex("@\"(?:[^\"]|\"\")*")]172 ErrorStringDoubleVerbatimUnterminated,173174 #[regex("@'(?:[^']|'')*")]175 ErrorStringSingleVerbatimUnterminated,176177 #[regex("@[^\"'\\s]\\S+")]178 ErrorStringMissingQuotes,179180 #[token("/*/")]181 ErrorCommentTooShort,182183 #[regex(r"/\*([^*]|\*[^/])+")]184 ErrorCommentUnterminated,185186 #[regex(r"[ \t\n\r]+")]187 Whitespace,188189 #[regex(r"//[^\r\n]*(\r\n|\n)?")]190 SingelLineSlashComment,191192 #[regex(r"#[^\r\n]*(\r\n|\n)?")]193 SingleLineHashComment,194195 #[regex(r"/\*([^*]|\*[^/])*\*/")]196 MultiLineComment,197198 #[error]199 Error,200201 ErrorPositionalAfterNamed,202203 Literal,204 Expr,205 Array,206 ArrayElem,207 Object,208 Field,209210 CompspecFor,211 CompspecIf,212213 Slice,214 FieldAccess,215 ObjectApply,216 FunctionCall,217 FunctionDef,218 BodyDef,219220 BinOp,221 UnaryOp,222 Local,223 ExprError,224 ExprAssert,225 ExprImport,226227 DefParam,228 DefParams,229230 DefArgs,231 DefNamedArg,232 DefPositionalArg,233234 Parened,235236 Root,237}238239impl SyntaxKind {240 pub fn is_trivia(self) -> bool {241 matches!(242 self,243 Self::Whitespace244 | Self::MultiLineComment245 | Self::SingelLineSlashComment246 | Self::SingleLineHashComment247 )248 }249}250251pub struct Lexer<'a> {252 inner: logos::Lexer<'a, SyntaxKind>,253}254255impl<'a> Lexer<'a> {256 pub fn new(input: &'a str) -> Self {257 Self {258 inner: SyntaxKind::lexer(input),259 }260 }261}262263impl<'a> Iterator for Lexer<'a> {264 type Item = Lexeme<'a>;265266 fn next(&mut self) -> Option<Self::Item> {267 let kind = self.inner.next()?;268 let text = self.inner.slice();269270 Some(Self::Item {271 kind,272 text,273 range: {274 let Range { start, end } = self.inner.span();275276 TextRange::new(277 TextSize::try_from(start).unwrap(),278 TextSize::try_from(end).unwrap(),279 )280 },281 })282 }283}284285#[derive(Clone, Copy)]286pub struct Lexeme<'i> {287 pub kind: SyntaxKind,288 pub text: &'i str,289 pub range: TextRange,290}291292pub fn lex(input: &str) -> Vec<Lexeme<'_>> {293 Lexer::new(input).collect()294}295296impl From<SyntaxKind> for rowan::SyntaxKind {297 fn from(kind: SyntaxKind) -> Self {298 Self(kind as u16)299 }300}301302use SyntaxKind::*;303304#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]305pub enum Lang {}306impl rowan::Language for Lang {307 type Kind = SyntaxKind;308 fn kind_from_raw(raw: rowan::SyntaxKind) -> Self::Kind {309 assert!(raw.0 <= Root as u16);310 unsafe { std::mem::transmute::<u16, SyntaxKind>(raw.0) }311 }312 fn kind_to_raw(kind: Self::Kind) -> rowan::SyntaxKind {313 kind.into()314 }315}