llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix attributes in self-closing tags

Browse Source
This commit is contained in:
Geens
2025-04-11 19:26:05 +02:00
parent 7e254b7f7e
commit bd3adf78e6
8 changed files with 54 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/xml_schema_validator/src/lib.rs
View File

@@ -368,7 +368,7 @@ mod diagnostic_tests {
// Check what type the token is // Check what type the token is
match &tokens[0] { match &tokens[0] {
Token::ElementOpen(element_open) => { Token::ElementOpenEnd(element_open) => {
println!("Successfully transitioned to ElementOpen: {:?}", element_open); println!("Successfully transitioned to ElementOpen: {:?}", element_open);
}, },
other => { other => {
@@ -529,7 +529,7 @@ mod diagnostic_tests {
assert!(!tokens.is_empty(), "StartOfFile should accept '<'"); assert!(!tokens.is_empty(), "StartOfFile should accept '<'");
let element_start = match &tokens[0] { let element_start = match &tokens[0] {
Token::ElementStart(es) => es.clone(), Token::ElementOpenStart(es) => es.clone(),
_ => panic!("Expected ElementStart"), _ => panic!("Expected ElementStart"),
}; };
@@ -537,7 +537,7 @@ mod diagnostic_tests {
assert!(!tokens.is_empty(), "ElementStart should accept 't'"); assert!(!tokens.is_empty(), "ElementStart should accept 't'");
let element_name = match &tokens[0] { let element_name = match &tokens[0] {
Token::ElementName(en) => en.clone(), Token::ElementOpenName(en) => en.clone(),
_ => panic!("Expected ElementName"), _ => panic!("Expected ElementName"),
}; };
@@ -562,7 +562,7 @@ mod diagnostic_tests {
// Verify transition to ElementOpen // Verify transition to ElementOpen
match &tokens[0] { match &tokens[0] {
Token::ElementOpen(_) => println!("Successfully transitioned to ElementOpen"), Token::ElementOpenEnd(_) => println!("Successfully transitioned to ElementOpen"),
other => panic!("Expected ElementOpen, got {:?}", other), other => panic!("Expected ElementOpen, got {:?}", other),
} }
} }

8
lib/xml_schema_validator/src/token/attribute_value.rs
View File

@@ -58,12 +58,12 @@ impl AttributeValue {
// No more required attributes, whitespace after attribute value, continue to next state // No more required attributes, whitespace after attribute value, continue to next state
return vec![ return vec![
Token::ElementOpen(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())), Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())),
Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes)) Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))
]; ];
} else if c == '>' { } else if c == '>' {
// End of opening tag // End of opening tag
return vec![Token::ElementOpen(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))]; return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))];
} else if c == '/' { } else if c == '/' {
// Beginning of self-closing tag // Beginning of self-closing tag
return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))]; return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))];
@@ -211,7 +211,7 @@ mod attribute_value_tests {
assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute"); assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute");
match &next_tokens[0] { match &next_tokens[0] {
Token::ElementOpen(_) => (), Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
} }
@@ -242,7 +242,7 @@ mod attribute_value_tests {
assert!(!next_tokens.is_empty(), "Should accept '>' after attribute"); assert!(!next_tokens.is_empty(), "Should accept '>' after attribute");
match &next_tokens[0] { match &next_tokens[0] {
Token::ElementOpen(element_open) => { Token::ElementOpenEnd(element_open) => {
// Verify that the id attribute is tracked // Verify that the id attribute is tracked
assert!(element_open.attributes_set.contains(&"id".to_string()), assert!(element_open.attributes_set.contains(&"id".to_string()),
"Attribute 'id' should be tracked"); "Attribute 'id' should be tracked");

35
lib/xml_schema_validator/src/token/element_open_name.rs
View File

@@ -34,10 +34,12 @@ impl ElementOpenName {
} }
} }
// No attributes, continue to element open or self-close // No attributes
return vec![ return vec![
Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element))), Token::AttributeName(AttributeName::new(Arc::clone(&self.element))),
Token::ElementSelfClose(ElementSelfClose::new()) Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))),
Token::ElementSelfClose(ElementSelfClose::new()),
Token::ElementOpenName(self),
]; ];
} else { } else {
// Whitespace in the middle of an element name is invalid // Whitespace in the middle of an element name is invalid
@@ -59,7 +61,7 @@ impl ElementOpenName {
} }
} }
return vec![Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element)))]; return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))];
} else { } else {
// '>' before full element name is invalid // '>' before full element name is invalid
return vec![]; return vec![];
@@ -79,7 +81,7 @@ impl ElementOpenName {
if self.element.name.starts_with(&new_buffer) { if self.element.name.starts_with(&new_buffer) {
// Still building the element name // Still building the element name
return vec![Token::ElementName(ElementOpenName { return vec![Token::ElementOpenName(ElementOpenName {
buffer: new_buffer, buffer: new_buffer,
element: Arc::clone(&self.element), element: Arc::clone(&self.element),
})]; })];
@@ -166,7 +168,7 @@ mod element_open_name_tests {
assert!(!next_tokens.is_empty(), "Should accept '>' after full element name"); assert!(!next_tokens.is_empty(), "Should accept '>' after full element name");
match &next_tokens[0] { match &next_tokens[0] {
Token::ElementOpen(_) => (), Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
} }
} }
@@ -182,10 +184,23 @@ mod element_open_name_tests {
let next_tokens = element_name.append(' '); let next_tokens = element_name.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after full element name"); assert!(!next_tokens.is_empty(), "Should accept space after full element name");
match &next_tokens[0] { assert!(next_tokens.len() == 4);
Token::ElementOpen(_) => (), assert!(next_tokens.iter()
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), .filter(|token| matches!(token, Token::ElementOpenName(_)))
} .next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::AttributeName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
.next()
.is_some());
} }
#[test] #[test]

2
lib/xml_schema_validator/src/token/element_open_start.rs
View File

@@ -17,7 +17,7 @@ impl ElementOpenStart {
pub fn append(self, c: char) -> Vec<Token> { pub fn append(self, c: char) -> Vec<Token> {
let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string()); let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string());
if let Ok(element_name) = element_name { if let Ok(element_name) = element_name {
vec![Token::ElementName(element_name)] vec![Token::ElementOpenName(element_name)]
} else { } else {
vec![] vec![]
} }

32
lib/xml_schema_validator/src/token/mod.rs
View File

@@ -27,9 +27,9 @@ pub use element_self_close::ElementSelfClose;
pub enum Token { pub enum Token {
ElementCloseName(ElementCloseName), ElementCloseName(ElementCloseName),
ElementCloseStart(ElementCloseStart), ElementCloseStart(ElementCloseStart),
ElementName(ElementOpenName), ElementOpenName(ElementOpenName),
ElementOpen(ElementOpenEnd), ElementOpenEnd(ElementOpenEnd),
ElementStart(ElementOpenStart), ElementOpenStart(ElementOpenStart),
EndOfFile(EndOfFile), EndOfFile(EndOfFile),
StartOfFile(StartOfFile), StartOfFile(StartOfFile),
TextContent(TextContent), TextContent(TextContent),
@@ -43,9 +43,9 @@ impl Token {
match self { match self {
Token::ElementCloseName(element_close_name) => element_close_name.append(c), Token::ElementCloseName(element_close_name) => element_close_name.append(c),
Token::ElementCloseStart(element_close_start) => element_close_start.append(c), Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
Token::ElementName(element_name) => element_name.append(c), Token::ElementOpenName(element_name) => element_name.append(c),
Token::ElementOpen(element_open) => element_open.append(c), Token::ElementOpenEnd(element_open) => element_open.append(c),
Token::ElementStart(element_start) => element_start.append(c), Token::ElementOpenStart(element_start) => element_start.append(c),
Token::EndOfFile(end_of_file) => end_of_file.append(c), Token::EndOfFile(end_of_file) => end_of_file.append(c),
Token::StartOfFile(start_of_file) => start_of_file.append(c), Token::StartOfFile(start_of_file) => start_of_file.append(c),
Token::TextContent(text_content) => text_content.append(c), Token::TextContent(text_content) => text_content.append(c),
@@ -100,7 +100,7 @@ mod tests {
assert_eq!(next_tokens.len(), 1, "Should have one transition"); assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] { match &next_tokens[0] {
Token::ElementStart(_) => (), Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]), _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
} }
@@ -113,19 +113,19 @@ mod tests {
#[test] #[test]
fn test_element_start_transition() { fn test_element_start_transition() {
let element = create_test_element(); let element = create_test_element();
let token = Token::ElementStart(ElementOpenStart::new(Arc::clone(&element))); let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
// Should accept first char of element name // Should accept first char of element name
let next_tokens = token.append('r'); let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should have one transition"); assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] { match &next_tokens[0] {
Token::ElementName(_) => (), Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]), _ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
} }
// Should reject invalid char // Should reject invalid char
let token = Token::ElementStart(ElementOpenStart::new(Arc::clone(&element))); let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
let next_tokens = token.append('x'); let next_tokens = token.append('x');
assert_eq!(next_tokens.len(), 0, "Should reject 'x'"); assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
} }
@@ -133,20 +133,20 @@ mod tests {
#[test] #[test]
fn test_element_name_transition() { fn test_element_name_transition() {
let element = create_test_element(); let element = create_test_element();
let token = Token::ElementName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap()); let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap());
// Continue building name // Continue building name
let next_tokens = token.append('e'); let next_tokens = token.append('e');
assert_eq!(next_tokens.len(), 1, "Should have one transition"); assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] { match &next_tokens[0] {
Token::ElementName(_) => (), Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]), _ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
} }
// Build complete name and end with > // Build complete name and end with >
// Remove the unused buffer // Remove the unused buffer
let mut current_tokens = vec![Token::ElementStart(ElementOpenStart::new(Arc::clone(&element)))]; let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))];
for c in "reasoning>".chars() { for c in "reasoning>".chars() {
let mut new_tokens = Vec::new(); let mut new_tokens = Vec::new();
@@ -162,7 +162,7 @@ mod tests {
// Check final state // Check final state
assert_eq!(current_tokens.len(), 1, "Should have one final transition"); assert_eq!(current_tokens.len(), 1, "Should have one final transition");
match &current_tokens[0] { match &current_tokens[0] {
Token::ElementOpen(_) => (), Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]), _ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
} }
} }
@@ -170,7 +170,7 @@ mod tests {
#[test] #[test]
fn test_element_open_end_transition() { fn test_element_open_end_transition() {
let element = create_test_element(); let element = create_test_element();
let token = Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&element))); let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
// Should accept text content in mixed element // Should accept text content in mixed element
let next_tokens = token.append('t'); let next_tokens = token.append('t');
@@ -182,7 +182,7 @@ mod tests {
} }
// Should accept closing tag start // Should accept closing tag start
let token = Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&element))); let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
let next_tokens = token.append('<'); let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'"); assert_eq!(next_tokens.len(), 1, "Should accept '<'");

4
lib/xml_schema_validator/src/token/start_of_file.rs
View File

@@ -14,7 +14,7 @@ impl StartOfFile {
pub fn append(self, c: char) -> Vec<Token> { pub fn append(self, c: char) -> Vec<Token> {
if '<' == c { if '<' == c {
vec![Token::ElementStart(ElementOpenStart::new(Arc::clone(&self.element)))] vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))]
} else { } else {
vec![] vec![]
} }
@@ -58,7 +58,7 @@ mod start_of_file_tests {
assert!(!next_tokens.is_empty(), "Should accept '<'"); assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] { match &next_tokens[0] {
Token::ElementStart(_) => (), Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]), _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
} }
} }

2
qwq_tokenizer_config.json
View File

@@ -227,7 +227,7 @@
"<|video_pad|>" "<|video_pad|>"
], ],
"bos_token": null, "bos_token": null,
"chat_template": "{%- for message in messages %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", "chat_template": "{%- for message in messages %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content }}\n {%- if not loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '\\n<|im_start|>assistant\\n' }}\n{%- endif %}",
"clean_up_tokenization_spaces": false, "clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>", "eos_token": "<|im_end|>",
"errors": "replace", "errors": "replace",

1
scripts/deploy.sh
View File

@@ -45,6 +45,7 @@ output=$(runpodctl create pod \
--env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \ --env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \
--env "SIA_QWQ_ENABLED=1" \ --env "SIA_QWQ_ENABLED=1" \
--env "JUPYTER_PASSWORD=1" \ --env "JUPYTER_PASSWORD=1" \
--communityCloud
) )
echo "$output" echo "$output"