Added char trie

lib/xml_schema_validator/src/char_trie.rs

@@ -0,0 +1,103 @@
+use crate::*;
+use pyo3::prelude::*;
+
+// Node in our character trie
+struct CharTrieNode {
+    // Store character and node pairs, keep sorted for binary search
+    children: Vec<(char, CharTrieNode)>,
+    // Store token IDs that are valid at this point
+    token_id: Option<Py<PyAny>>,
+}
+
+impl CharTrieNode {
+    fn new() -> Self {
+        Self {
+            children: Vec::new(),
+            token_id: None,
+        }
+    }
+    
+    // Add a child for character c, maintaining sorted order
+    fn add_child(&mut self, c: char) -> &mut CharTrieNode {
+        match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
+            Ok(index) => &mut self.children[index].1,
+            Err(index) => {
+                // Insert at the right position to maintain sort order
+                self.children.insert(index, (c, CharTrieNode::new()));
+                &mut self.children[index].1
+            }
+        }
+    }
+    
+    // Add a token id to this node
+    fn add_token(&mut self, token_id: Py<PyAny>) {
+        self.token_id = Some(token_id);
+    }
+
+    pub fn clone_ref(&self, py: Python<'_>) -> Self {
+        Self {
+            children: self.children.iter().map(|(ch, node)| (*ch, node.clone_ref(py))).collect(),
+            token_id: self.token_id.iter().map(|token_id| token_id.clone_ref(py)).nth(0),
+        }
+    }
+}
+
+// The main trie structure
+pub struct CharTrie {
+    root: CharTrieNode,
+}
+
+impl CharTrie {
+    pub fn new() -> Self {
+        Self {
+            root: CharTrieNode::new(),
+        }
+    }
+    
+    // Insert a token's character sequence into the trie
+    pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
+        let mut current = &mut self.root;
+        
+        // Add each character in the token text
+        for c in token_text.chars() {
+            current = current.add_child(c);
+        }
+        
+        // Mark this node as representing token_id
+        current.add_token(token_id);
+    }
+    
+    // Find all tokens that could be invalid given the current XML state
+    pub fn find_invalid_tokens(&self, py: Python<'_>, xml_validator: &XmlSchemaValidator, token_map: &Vec<(Py<PyAny>, String)>) -> Vec<Py<PyAny>> {
+        let mut invalid_tokens = Vec::new();
+        
+        for (token_id, token_text) in token_map {
+            // Check if this token would make the XML invalid
+            if !self.is_valid_continuation(xml_validator, token_text) {
+                invalid_tokens.push(token_id.clone_ref(py));
+            }
+        }
+        
+        invalid_tokens
+    }
+    
+    // Check if a token would be a valid continuation
+    fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
+        // First, quick check with just the first character
+        if let Some(first_char) = token_text.chars().next() {
+            if !xml_validator.can_continue_with(first_char) {
+                return false;
+            }
+        }
+        
+        // If first character is valid, try the full token
+        let mut validator_clone = xml_validator.clone();
+        validator_clone.append(token_text).is_ok()
+    }
+
+    pub fn clone_ref(&self, py: Python<'_>) -> Self {
+        Self {
+            root: self.root.clone_ref(py),
+        }
+    }
+}