UPDATE: Implement Insert/Delete Cells

base/src/expressions/lexer/marked_token.rs

@@ -1,3 +1,5 @@
+#![deny(missing_docs)]
+
 use serde::{Deserialize, Serialize};
 
 use crate::expressions::token;
@@ -9,8 +11,11 @@ use super::{Lexer, LexerMode};
 /// A MarkedToken is a token together with its position on a formula
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct MarkedToken {
+    /// Token type (see [token::TokenType])
     pub token: token::TokenType,
+    /// Position of the start of the token (in bytes)
     pub start: i32,
+    /// Position of the end of the token (in bytes)
     pub end: i32,
 }
 
@@ -19,7 +24,7 @@ pub struct MarkedToken {
 /// # Examples
 /// ```
 /// use ironcalc_base::expressions::{
-///      lexer::util::{get_tokens, MarkedToken},
+///      lexer::marked_token::{get_tokens, MarkedToken},
 ///      token::{OpSum, TokenType},
 /// };
 ///

base/src/expressions/lexer/mod.rs

@@ -1,3 +1,5 @@
+#![deny(missing_docs)]
+
 //! A tokenizer for spreadsheet formulas.
 //!
 //! This is meant to feed a formula parser.
@@ -7,8 +9,10 @@
 //! It supports two working modes:
 //!
 //! 1. A1 or display mode
+//!
 //!    This is for user formulas. References are like `D4`, `D$4` or `F5:T10`
 //! 2. R1C1, internal or runtime mode
+//!
 //!    A reference like R1C1 refers to $A$1 and R3C4 to $D$4
 //!    R[2]C[5] refers to a cell two rows below and five columns to the right
 //!    It uses the 'en' locale and language.
@@ -55,7 +59,8 @@ use super::token::{Error, TokenType};
 use super::types::*;
 use super::utils;
 
-pub mod util;
+/// Returns an iterator over tokens together with their position in the byte string.
+pub mod marked_token;
 
 #[cfg(test)]
 mod test;
@@ -63,17 +68,28 @@ mod test;
 mod ranges;
 mod structured_references;
 
+/// This is the TokenType we return if we cannot recognize a token
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct LexerError {
+    /// Position of the beginning of the token in the byte string.
     pub position: usize,
+    /// Message describing what we think the error is.
     pub message: String,
 }
 
 pub(super) type Result<T> = std::result::Result<T, LexerError>;
 
+/// Whether we try to parse formulas in A1 mode or in the internal R1C1 mode
 #[derive(Clone, PartialEq, Eq)]
 pub enum LexerMode {
+    /// Cell references are written `=S34`. This is the display mode
     A1,
+    ///    R1C1, internal or runtime mode
+    ///
+    ///    A reference like R1C1 refers to $A$1 and R3C4 to $D$4
+    ///    R[2]C[5] refers to a cell two rows below and five columns to the right
+    ///    It uses the 'en' locale and language.
+    ///    This is used internally at runtime.
     R1C1,
 }
 

base/src/expressions/lexer/test/mod.rs

@@ -1,6 +1,6 @@
 mod test_common;
 mod test_language;
 mod test_locale;
+mod test_marked_token;
 mod test_ranges;
 mod test_tables;
-mod test_util;

base/src/expressions/lexer/test/test_marked_token.rs

@@ -1,5 +1,5 @@
 use crate::expressions::{
-    lexer::util::get_tokens,
+    lexer::marked_token::{get_tokens, MarkedToken},
     token::{OpCompare, OpSum, TokenType},
 };
 
@@ -22,6 +22,29 @@ fn test_get_tokens() {
     assert_eq!(l.end, 10);
 }
 
+#[test]
+fn chinese_characters() {
+    let formula = "\"你好\" & \"世界！\"";
+    let marked_tokens = get_tokens(formula);
+    assert_eq!(marked_tokens.len(), 3);
+    let first_t = MarkedToken {
+        token: TokenType::String("你好".to_string()),
+        start: 0,
+        end: 4,
+    };
+    let second_t = MarkedToken {
+        token: TokenType::And,
+        start: 4,
+        end: 6,
+    };
+    let third_t = MarkedToken {
+        token: TokenType::String("世界！".to_string()),
+        start: 6,
+        end: 12,
+    };
+    assert_eq!(marked_tokens, vec![first_t, second_t, third_t]);
+}
+
 #[test]
 fn test_simple_tokens() {
     assert_eq!(