git.delta.rocks / jrsonnet / refs/commits / 94c438bdfffb

1use core::ops::Range;23use logos::Logos;4// use rowan::{TextRange, TextSize};56use crate::{7	generated::syntax_kinds::SyntaxKind,8	string_block::{lex_str_block, StringBlockError},9	Span,10};1112pub struct Lexer<'a> {13	inner: logos::Lexer<'a, SyntaxKind>,14}1516impl<'a> Lexer<'a> {17	pub fn new(input: &'a str) -> Self {18		Self {19			inner: SyntaxKind::lexer(input),20		}21	}22}2324impl<'a> Iterator for Lexer<'a> {25	type Item = Lexeme<'a>;2627	fn next(&mut self) -> Option<Self::Item> {28		use SyntaxKind::*;2930		let mut kind = self.inner.next()?;31		let text = self.inner.slice();3233		if kind == Ok(STRING_BLOCK) {34			// We use custom lexer, which skips enough bytes, but not returns error35			// Instead we should call lexer again to verify if there is something wrong with string block36			let mut lexer = logos::Lexer::<SyntaxKind>::new(text);37			// In kinds, string blocks is parsed at least as `|||`38			lexer.bump(3);39			let res = lex_str_block(&mut lexer);40			let next = lexer.next();41			assert!(next.is_none(), "str_block is lexed");42			match res {43				Ok(()) => {}44				Err(e) => {45					kind = Ok(match e {46						StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,47						StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,48						StringBlockError::MissingTermination => {49							ERROR_STRING_BLOCK_MISSING_TERMINATION50						}51						StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,52					});53				}54			}55		}5657		Some(Self::Item {58			kind: kind.unwrap_or(SyntaxKind::LEXING_ERROR),59			text,60			range: {61				let Range { start, end } = self.inner.span();6263				Span(start as u32, end as u32)64			},65		})66	}67}6869#[derive(Clone, Copy, Debug)]70pub struct Lexeme<'s> {71	pub kind: SyntaxKind,72	pub text: &'s str,73	pub range: Span,74}7576pub fn lex(input: &str) -> Vec<Lexeme<'_>> {77	Lexer::new(input).collect()78}