git.delta.rocks / jrsonnet / refs/commits / d349b9ef06a9

1use core::ops::Range;2use std::convert::TryFrom;34use logos::Logos;5use rowan::{TextRange, TextSize};67use crate::{8	string_block::{lex_str_block, StringBlockError},9	SyntaxKind,10};1112pub struct Lexer<'a> {13	inner: logos::Lexer<'a, SyntaxKind>,14}1516impl<'a> Lexer<'a> {17	pub fn new(input: &'a str) -> Self {18		Self {19			inner: SyntaxKind::lexer(input),20		}21	}22}2324impl<'a> Iterator for Lexer<'a> {25	type Item = Lexeme<'a>;2627	fn next(&mut self) -> Option<Self::Item> {28		use SyntaxKind::*;2930		let mut kind = self.inner.next()?;31		let text = self.inner.slice();3233		if kind == Ok(STRING_BLOCK) {34			// We use custom lexer, which skips enough bytes, but not returns error35			// Instead we should call lexer again to verify if there is something wrong with string block36			let mut lexer = logos::Lexer::<SyntaxKind>::new(text);37			// In kinds, string blocks is parsed at least as `|||`38			lexer.bump(3);39			let res = lex_str_block(&mut lexer);40			debug_assert!(lexer.next().is_none(), "str_block is lexed");41			match res {42				Ok(_) => {}43				Err(e) => {44					kind = Ok(match e {45						StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,46						StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,47						StringBlockError::MissingTermination => {48							ERROR_STRING_BLOCK_MISSING_TERMINATION49						}50						StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,51					})52				}53			}54		}5556		Some(Self::Item {57			kind: kind.unwrap_or(SyntaxKind::LEXING_ERROR),58			text,59			range: {60				let Range { start, end } = self.inner.span();6162				TextRange::new(63					TextSize::try_from(start).unwrap(),64					TextSize::try_from(end).unwrap(),65				)66			},67		})68	}69}7071#[derive(Clone, Copy, Debug)]72pub struct Lexeme<'i> {73	pub kind: SyntaxKind,74	pub text: &'i str,75	pub range: TextRange,76}7778pub fn lex(input: &str) -> Vec<Lexeme<'_>> {79	Lexer::new(input).collect()80}