Fixed attribute parsing
This commit is contained in:
@@ -1,103 +1,116 @@
|
||||
use crate::*;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
// Node in our character trie
|
||||
struct CharTrieNode {
|
||||
// Store character and node pairs, keep sorted for binary search
|
||||
children: Vec<(char, CharTrieNode)>,
|
||||
// Store token IDs that are valid at this point
|
||||
token_id: Option<Py<PyAny>>,
|
||||
}
|
||||
|
||||
impl CharTrieNode {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
children: Vec::new(),
|
||||
token_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
// Add a child for character c, maintaining sorted order
|
||||
fn add_child(&mut self, c: char) -> &mut CharTrieNode {
|
||||
match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
|
||||
Ok(index) => &mut self.children[index].1,
|
||||
Err(index) => {
|
||||
// Insert at the right position to maintain sort order
|
||||
self.children.insert(index, (c, CharTrieNode::new()));
|
||||
&mut self.children[index].1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add a token id to this node
|
||||
fn add_token(&mut self, token_id: Py<PyAny>) {
|
||||
self.token_id = Some(token_id);
|
||||
}
|
||||
|
||||
pub fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
Self {
|
||||
children: self.children.iter().map(|(ch, node)| (*ch, node.clone_ref(py))).collect(),
|
||||
token_id: self.token_id.iter().map(|token_id| token_id.clone_ref(py)).nth(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The main trie structure
|
||||
pub struct CharTrie {
|
||||
root: CharTrieNode,
|
||||
}
|
||||
|
||||
impl CharTrie {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
root: CharTrieNode::new(),
|
||||
}
|
||||
}
|
||||
|
||||
// Insert a token's character sequence into the trie
|
||||
pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
|
||||
let mut current = &mut self.root;
|
||||
|
||||
// Add each character in the token text
|
||||
for c in token_text.chars() {
|
||||
current = current.add_child(c);
|
||||
}
|
||||
|
||||
// Mark this node as representing token_id
|
||||
current.add_token(token_id);
|
||||
}
|
||||
|
||||
// Find all tokens that could be invalid given the current XML state
|
||||
pub fn find_invalid_tokens(&self, py: Python<'_>, xml_validator: &XmlSchemaValidator, token_map: &Vec<(Py<PyAny>, String)>) -> Vec<Py<PyAny>> {
|
||||
let mut invalid_tokens = Vec::new();
|
||||
|
||||
for (token_id, token_text) in token_map {
|
||||
// Check if this token would make the XML invalid
|
||||
if !self.is_valid_continuation(xml_validator, token_text) {
|
||||
invalid_tokens.push(token_id.clone_ref(py));
|
||||
}
|
||||
}
|
||||
|
||||
invalid_tokens
|
||||
}
|
||||
|
||||
// Check if a token would be a valid continuation
|
||||
fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
|
||||
// First, quick check with just the first character
|
||||
if let Some(first_char) = token_text.chars().next() {
|
||||
if !xml_validator.can_continue_with(first_char) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// If first character is valid, try the full token
|
||||
let mut validator_clone = xml_validator.clone();
|
||||
validator_clone.append(token_text).is_ok()
|
||||
}
|
||||
|
||||
pub fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
Self {
|
||||
root: self.root.clone_ref(py),
|
||||
}
|
||||
}
|
||||
}
|
||||
use crate::*;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
// Node in our character trie
|
||||
struct CharTrieNode {
|
||||
// Store character and node pairs, keep sorted for binary search
|
||||
children: Vec<(char, CharTrieNode)>,
|
||||
// Store token IDs that are valid at this point
|
||||
token_id: Option<Py<PyAny>>,
|
||||
}
|
||||
|
||||
impl CharTrieNode {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
children: Vec::new(),
|
||||
token_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
// Add a child for character c, maintaining sorted order
|
||||
fn add_child(&mut self, c: char) -> &mut CharTrieNode {
|
||||
match self.children.binary_search_by_key(&c, |(ch, _)| *ch) {
|
||||
Ok(index) => &mut self.children[index].1,
|
||||
Err(index) => {
|
||||
// Insert at the right position to maintain sort order
|
||||
self.children.insert(index, (c, CharTrieNode::new()));
|
||||
&mut self.children[index].1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add a token id to this node
|
||||
fn add_token(&mut self, token_id: Py<PyAny>) {
|
||||
self.token_id = Some(token_id);
|
||||
}
|
||||
|
||||
pub fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
Self {
|
||||
children: self
|
||||
.children
|
||||
.iter()
|
||||
.map(|(ch, node)| (*ch, node.clone_ref(py)))
|
||||
.collect(),
|
||||
token_id: self
|
||||
.token_id
|
||||
.iter()
|
||||
.map(|token_id| token_id.clone_ref(py))
|
||||
.nth(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The main trie structure
|
||||
pub struct CharTrie {
|
||||
root: CharTrieNode,
|
||||
}
|
||||
|
||||
impl CharTrie {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
root: CharTrieNode::new(),
|
||||
}
|
||||
}
|
||||
|
||||
// Insert a token's character sequence into the trie
|
||||
pub fn insert(&mut self, token_text: &str, token_id: Py<PyAny>) {
|
||||
let mut current = &mut self.root;
|
||||
|
||||
// Add each character in the token text
|
||||
for c in token_text.chars() {
|
||||
current = current.add_child(c);
|
||||
}
|
||||
|
||||
// Mark this node as representing token_id
|
||||
current.add_token(token_id);
|
||||
}
|
||||
|
||||
// Find all tokens that could be invalid given the current XML state
|
||||
pub fn find_invalid_tokens(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
xml_validator: &XmlSchemaValidator,
|
||||
token_map: &Vec<(Py<PyAny>, String)>,
|
||||
) -> Vec<Py<PyAny>> {
|
||||
let mut invalid_tokens = Vec::new();
|
||||
|
||||
for (token_id, token_text) in token_map {
|
||||
// Check if this token would make the XML invalid
|
||||
if !self.is_valid_continuation(xml_validator, token_text) {
|
||||
invalid_tokens.push(token_id.clone_ref(py));
|
||||
}
|
||||
}
|
||||
|
||||
invalid_tokens
|
||||
}
|
||||
|
||||
// Check if a token would be a valid continuation
|
||||
fn is_valid_continuation(&self, xml_validator: &XmlSchemaValidator, token_text: &str) -> bool {
|
||||
// First, quick check with just the first character
|
||||
if let Some(first_char) = token_text.chars().next() {
|
||||
if !xml_validator.can_continue_with(first_char) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// If first character is valid, try the full token
|
||||
let mut validator_clone = xml_validator.clone();
|
||||
validator_clone.append(token_text).is_ok()
|
||||
}
|
||||
|
||||
pub fn clone_ref(&self, py: Python<'_>) -> Self {
|
||||
Self {
|
||||
root: self.root.clone_ref(py),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user