git.delta.rocks / jrsonnet / refs/commits / afca77d1e35b

1use core::ops::Range;2use std::convert::TryFrom;34use logos::Logos;5use rowan::{TextRange, TextSize};67use crate::{8	string_block::{lex_str_block, StringBlockError},9	SyntaxKind,10};1112pub struct Lexer<'a> {13	inner: logos::Lexer<'a, SyntaxKind>,14}1516impl<'a> Lexer<'a> {17	pub fn new(input: &'a str) -> Self {18		Self {19			inner: SyntaxKind::lexer(input),20		}21	}22}2324impl<'a> Iterator for Lexer<'a> {25	type Item = Lexeme<'a>;2627	fn next(&mut self) -> Option<Self::Item> {28		use SyntaxKind::*;2930		let mut kind = self.inner.next()?;31		let text = self.inner.slice();3233		if kind == Ok(STRING_BLOCK) {34			// We use custom lexer, which skips enough bytes, but not returns error35			// Instead we should call lexer again to verify if there is something wrong with string block36			let mut lexer = logos::Lexer::<SyntaxKind>::new(text);37			// In kinds, string blocks is parsed at least as `|||`38			lexer.bump(3);39			let res = lex_str_block(&mut lexer);40			let next = lexer.next();41			assert!(next.is_none(), "str_block is lexed");42			match res {43				Ok(()) => {}44				Err(e) => {45					kind = Ok(match e {46						StringBlockError::UnexpectedEnd => ERROR_STRING_BLOCK_UNEXPECTED_END,47						StringBlockError::MissingNewLine => ERROR_STRING_BLOCK_MISSING_NEW_LINE,48						StringBlockError::MissingTermination => {49							ERROR_STRING_BLOCK_MISSING_TERMINATION50						}51						StringBlockError::MissingIndent => ERROR_STRING_BLOCK_MISSING_INDENT,52					});53				}54			}55		}5657		Some(Self::Item {58			kind: kind.unwrap_or(SyntaxKind::LEXING_ERROR),59			text,60			range: {61				let Range { start, end } = self.inner.span();6263				TextRange::new(64					TextSize::try_from(start).unwrap(),65					TextSize::try_from(end).unwrap(),66				)67			},68		})69	}70}7172#[derive(Clone, Copy, Debug)]73pub struct Lexeme<'i> {74	pub kind: SyntaxKind,75	pub text: &'i str,76	pub range: TextRange,77}7879pub fn lex(input: &str) -> Vec<Lexeme<'_>> {80	Lexer::new(input).collect()81}