git.delta.rocks / jrsonnet / refs/commits / b89fdd32bd3a

1use core::ops::Range;23use logos::Logos;45// use rowan::{TextRange, TextSize};6use crate::{7	Span,8	generated::syntax_kinds::SyntaxKind,9	string_block::{StringBlockError, lex_str_block},10};1112pub struct Lexer<'a> {13	inner: logos::Lexer<'a, SyntaxKind>,14}1516impl<'a> Lexer<'a> {17	pub fn new(input: &'a str) -> Self {18		Self {19			inner: SyntaxKind::lexer(input),20		}21	}22}2324impl<'a> Iterator for Lexer<'a> {25	type Item = Lexeme<'a>;2627	fn next(&mut self) -> Option<Self::Item> {28		use SyntaxKind::*;2930		let mut kind = self.inner.next()?;31		let text = self.inner.slice();3233		if kind == Ok(STRING_BLOCK) {34			// We use custom lexer, which skips enough bytes, but not returns error35			// Instead we should call lexer again to verify if there is something wrong with string block36			let mut lexer = logos::Lexer::<SyntaxKind>::new(text);37			// In kinds, string blocks is parsed at least as `|||`38			lexer.bump(3);39			let res = lex_str_block(&mut lexer);40			let next = lexer.next();41			assert!(next.is_none(), "str_block is lexed");42			match res {43				Ok(()) => {}44				Err(e) => {45					kind = Ok(match e {46						StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,47						StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,48						StringBlockError::MissingTermination => {49							ERROR_STRING_BLOCK_MISSING_TERMINATION50						}51						StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,52					});53				}54			}55		}5657		Some(Self::Item {58			kind: kind.unwrap_or(SyntaxKind::LEXING_ERROR),59			text,60			range: {61				let Range { start, end } = self.inner.span();6263				Span(64					u32::try_from(start).expect("code size is limited by 4gb"),65					u32::try_from(end).expect("code size is limited by 4gb"),66				)67			},68		})69	}70}7172#[derive(Clone, Copy, Debug)]73pub struct Lexeme<'s> {74	pub kind: SyntaxKind,75	pub text: &'s str,76	pub range: Span,77}7879pub fn lex(input: &str) -> Vec<Lexeme<'_>> {80	Lexer::new(input).collect()81}