1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

use std::fmt;

use crate::expressions::token;
use crate::language::get_language;
use crate::locale::get_locale;

use super::{Lexer, LexerMode};

/// A MarkedToken is a token together with its position on a formula
#[derive(Debug, PartialEq)]
pub struct MarkedToken {
    pub token: token::TokenType,
    pub start: i32,
    pub end: i32,
}

/// Returns a list of marked tokens for a formula
///
/// # Examples
/// ```
/// use ironcalc_base::expressions::{
///      lexer::util::{get_tokens, MarkedToken},
///      token::{OpSum, TokenType},
/// };
///
/// let marked_tokens = get_tokens("A1+1");
/// let first_t = MarkedToken {
///     token: TokenType::Reference {
///         sheet: None,
///         row: 1,
///         column: 1,
///         absolute_column: false,
///         absolute_row: false,
///     },
///     start: 0,
///     end: 2,
/// };
/// let second_t = MarkedToken {
///     token: TokenType::Addition(OpSum::Add),
///     start:2,
///     end: 3
/// };
/// let third_t = MarkedToken {
///     token: TokenType::Number(1.0),
///     start:3,
///     end: 4
/// };
/// assert_eq!(marked_tokens, vec![first_t, second_t, third_t]);
/// ```
pub fn get_tokens(formula: &str) -> Vec<MarkedToken> {
    let mut tokens = Vec::new();
    let mut lexer = Lexer::new(
        formula,
        LexerMode::A1,
        get_locale("en").expect(""),
        get_language("en").expect(""),
    );
    let mut start = lexer.get_position();
    let mut next_token = lexer.next_token();
    let mut end = lexer.get_position();
    loop {
        match next_token {
            token::TokenType::EOF => {
                break;
            }
            _ => {
                tokens.push(MarkedToken {
                    start,
                    end,
                    token: next_token,
                });
                start = lexer.get_position();
                next_token = lexer.next_token();
                end = lexer.get_position();
            }
        }
    }
    tokens
}

impl fmt::Display for MarkedToken {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        write!(fmt, "{}", self.token)
    }
}