llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix attributes in self-closing tags

Browse Source
This commit is contained in:
Geens
2025-04-11 19:26:05 +02:00
parent 7e254b7f7e
commit bd3adf78e6
8 changed files with 54 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/xml_schema_validator/src/lib.rs
View File

@@ -368,7 +368,7 @@ mod diagnostic_tests {
// Check what type the token is
match &tokens[0] {
Token::ElementOpen(element_open) => {
Token::ElementOpenEnd(element_open) => {
println!("Successfully transitioned to ElementOpen: {:?}", element_open);
},
other => {
@@ -529,7 +529,7 @@ mod diagnostic_tests {
assert!(!tokens.is_empty(), "StartOfFile should accept '<'");
let element_start = match &tokens[0] {
Token::ElementStart(es) => es.clone(),
Token::ElementOpenStart(es) => es.clone(),
_ => panic!("Expected ElementStart"),
};
@@ -537,7 +537,7 @@ mod diagnostic_tests {
assert!(!tokens.is_empty(), "ElementStart should accept 't'");
let element_name = match &tokens[0] {
Token::ElementName(en) => en.clone(),
Token::ElementOpenName(en) => en.clone(),
_ => panic!("Expected ElementName"),
};
@@ -562,7 +562,7 @@ mod diagnostic_tests {
// Verify transition to ElementOpen
match &tokens[0] {
Token::ElementOpen(_) => println!("Successfully transitioned to ElementOpen"),
Token::ElementOpenEnd(_) => println!("Successfully transitioned to ElementOpen"),
other => panic!("Expected ElementOpen, got {:?}", other),
}
}

8
lib/xml_schema_validator/src/token/attribute_value.rs
View File

@@ -58,12 +58,12 @@ impl AttributeValue {
// No more required attributes, whitespace after attribute value, continue to next state
return vec![
Token::ElementOpen(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())),
Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())),
Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))
];
} else if c == '>' {
// End of opening tag
return vec![Token::ElementOpen(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))];
return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))];
} else if c == '/' {
// Beginning of self-closing tag
return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))];
@@ -211,7 +211,7 @@ mod attribute_value_tests {
assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute");
match &next_tokens[0] {
Token::ElementOpen(_) => (),
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
@@ -242,7 +242,7 @@ mod attribute_value_tests {
assert!(!next_tokens.is_empty(), "Should accept '>' after attribute");
match &next_tokens[0] {
Token::ElementOpen(element_open) => {
Token::ElementOpenEnd(element_open) => {
// Verify that the id attribute is tracked
assert!(element_open.attributes_set.contains(&"id".to_string()),
"Attribute 'id' should be tracked");

35
lib/xml_schema_validator/src/token/element_open_name.rs
View File

@@ -34,10 +34,12 @@ impl ElementOpenName {
}
}
// No attributes, continue to element open or self-close
// No attributes
return vec![
Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element))),
Token::ElementSelfClose(ElementSelfClose::new())
Token::AttributeName(AttributeName::new(Arc::clone(&self.element))),
Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))),
Token::ElementSelfClose(ElementSelfClose::new()),
Token::ElementOpenName(self),
];
} else {
// Whitespace in the middle of an element name is invalid
@@ -59,7 +61,7 @@ impl ElementOpenName {
}
}
return vec![Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element)))];
return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))];
} else {
// '>' before full element name is invalid
return vec![];
@@ -79,7 +81,7 @@ impl ElementOpenName {
if self.element.name.starts_with(&new_buffer) {
// Still building the element name
return vec![Token::ElementName(ElementOpenName {
return vec![Token::ElementOpenName(ElementOpenName {
buffer: new_buffer,
element: Arc::clone(&self.element),
})];
@@ -166,7 +168,7 @@ mod element_open_name_tests {
assert!(!next_tokens.is_empty(), "Should accept '>' after full element name");
match &next_tokens[0] {
Token::ElementOpen(_) => (),
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
}
@@ -182,10 +184,23 @@ mod element_open_name_tests {
let next_tokens = element_name.append(' ');
assert!(!next_tokens.is_empty(), "Should accept space after full element name");
match &next_tokens[0] {
Token::ElementOpen(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]),
}
assert!(next_tokens.len() == 4);
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::AttributeName(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementOpenEnd(_)))
.next()
.is_some());
assert!(next_tokens.iter()
.filter(|token| matches!(token, Token::ElementSelfClose(_)))
.next()
.is_some());
}
#[test]

2
lib/xml_schema_validator/src/token/element_open_start.rs
View File

@@ -17,7 +17,7 @@ impl ElementOpenStart {
pub fn append(self, c: char) -> Vec<Token> {
let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string());
if let Ok(element_name) = element_name {
vec![Token::ElementName(element_name)]
vec![Token::ElementOpenName(element_name)]
} else {
vec![]
}

32
lib/xml_schema_validator/src/token/mod.rs
View File

@@ -27,9 +27,9 @@ pub use element_self_close::ElementSelfClose;
pub enum Token {
ElementCloseName(ElementCloseName),
ElementCloseStart(ElementCloseStart),
ElementName(ElementOpenName),
ElementOpen(ElementOpenEnd),
ElementStart(ElementOpenStart),
ElementOpenName(ElementOpenName),
ElementOpenEnd(ElementOpenEnd),
ElementOpenStart(ElementOpenStart),
EndOfFile(EndOfFile),
StartOfFile(StartOfFile),
TextContent(TextContent),
@@ -43,9 +43,9 @@ impl Token {
match self {
Token::ElementCloseName(element_close_name) => element_close_name.append(c),
Token::ElementCloseStart(element_close_start) => element_close_start.append(c),
Token::ElementName(element_name) => element_name.append(c),
Token::ElementOpen(element_open) => element_open.append(c),
Token::ElementStart(element_start) => element_start.append(c),
Token::ElementOpenName(element_name) => element_name.append(c),
Token::ElementOpenEnd(element_open) => element_open.append(c),
Token::ElementOpenStart(element_start) => element_start.append(c),
Token::EndOfFile(end_of_file) => end_of_file.append(c),
Token::StartOfFile(start_of_file) => start_of_file.append(c),
Token::TextContent(text_content) => text_content.append(c),
@@ -100,7 +100,7 @@ mod tests {
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementStart(_) => (),
Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
}
@@ -113,19 +113,19 @@ mod tests {
#[test]
fn test_element_start_transition() {
let element = create_test_element();
let token = Token::ElementStart(ElementOpenStart::new(Arc::clone(&element)));
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
// Should accept first char of element name
let next_tokens = token.append('r');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementName(_) => (),
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Should reject invalid char
let token = Token::ElementStart(ElementOpenStart::new(Arc::clone(&element)));
let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)));
let next_tokens = token.append('x');
assert_eq!(next_tokens.len(), 0, "Should reject 'x'");
}
@@ -133,20 +133,20 @@ mod tests {
#[test]
fn test_element_name_transition() {
let element = create_test_element();
let token = Token::ElementName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap());
let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap());
// Continue building name
let next_tokens = token.append('e');
assert_eq!(next_tokens.len(), 1, "Should have one transition");
match &next_tokens[0] {
Token::ElementName(_) => (),
Token::ElementOpenName(_) => (),
_ => panic!("Expected ElementName, got {:?}", next_tokens[0]),
}
// Build complete name and end with >
// Remove the unused buffer
let mut current_tokens = vec![Token::ElementStart(ElementOpenStart::new(Arc::clone(&element)))];
let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))];
for c in "reasoning>".chars() {
let mut new_tokens = Vec::new();
@@ -162,7 +162,7 @@ mod tests {
// Check final state
assert_eq!(current_tokens.len(), 1, "Should have one final transition");
match &current_tokens[0] {
Token::ElementOpen(_) => (),
Token::ElementOpenEnd(_) => (),
_ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]),
}
}
@@ -170,7 +170,7 @@ mod tests {
#[test]
fn test_element_open_end_transition() {
let element = create_test_element();
let token = Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&element)));
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
// Should accept text content in mixed element
let next_tokens = token.append('t');
@@ -182,7 +182,7 @@ mod tests {
}
// Should accept closing tag start
let token = Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&element)));
let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element)));
let next_tokens = token.append('<');
assert_eq!(next_tokens.len(), 1, "Should accept '<'");

4
lib/xml_schema_validator/src/token/start_of_file.rs
View File

@@ -14,7 +14,7 @@ impl StartOfFile {
pub fn append(self, c: char) -> Vec<Token> {
if '<' == c {
vec![Token::ElementStart(ElementOpenStart::new(Arc::clone(&self.element)))]
vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))]
} else {
vec![]
}
@@ -58,7 +58,7 @@ mod start_of_file_tests {
assert!(!next_tokens.is_empty(), "Should accept '<'");
match &next_tokens[0] {
Token::ElementStart(_) => (),
Token::ElementOpenStart(_) => (),
_ => panic!("Expected ElementStart, got {:?}", next_tokens[0]),
}
}

2
qwq_tokenizer_config.json
View File

@@ -227,7 +227,7 @@
"<|video_pad|>"
],
"bos_token": null,
"chat_template": "{%- for message in messages %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}",
"chat_template": "{%- for message in messages %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content }}\n {%- if not loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '\\n<|im_start|>assistant\\n' }}\n{%- endif %}",
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",

1
scripts/deploy.sh
View File

@@ -45,6 +45,7 @@ output=$(runpodctl create pod \
--env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \
--env "SIA_QWQ_ENABLED=1" \
--env "JUPYTER_PASSWORD=1" \
--communityCloud
)
echo "$output"