diff --git a/lib/xml_schema_validator/src/lib.rs b/lib/xml_schema_validator/src/lib.rs index a432809..de3d1c0 100644 --- a/lib/xml_schema_validator/src/lib.rs +++ b/lib/xml_schema_validator/src/lib.rs @@ -368,7 +368,7 @@ mod diagnostic_tests { // Check what type the token is match &tokens[0] { - Token::ElementOpen(element_open) => { + Token::ElementOpenEnd(element_open) => { println!("Successfully transitioned to ElementOpen: {:?}", element_open); }, other => { @@ -529,7 +529,7 @@ mod diagnostic_tests { assert!(!tokens.is_empty(), "StartOfFile should accept '<'"); let element_start = match &tokens[0] { - Token::ElementStart(es) => es.clone(), + Token::ElementOpenStart(es) => es.clone(), _ => panic!("Expected ElementStart"), }; @@ -537,7 +537,7 @@ mod diagnostic_tests { assert!(!tokens.is_empty(), "ElementStart should accept 't'"); let element_name = match &tokens[0] { - Token::ElementName(en) => en.clone(), + Token::ElementOpenName(en) => en.clone(), _ => panic!("Expected ElementName"), }; @@ -562,7 +562,7 @@ mod diagnostic_tests { // Verify transition to ElementOpen match &tokens[0] { - Token::ElementOpen(_) => println!("Successfully transitioned to ElementOpen"), + Token::ElementOpenEnd(_) => println!("Successfully transitioned to ElementOpen"), other => panic!("Expected ElementOpen, got {:?}", other), } } diff --git a/lib/xml_schema_validator/src/token/attribute_value.rs b/lib/xml_schema_validator/src/token/attribute_value.rs index 2f604e9..6f114f7 100644 --- a/lib/xml_schema_validator/src/token/attribute_value.rs +++ b/lib/xml_schema_validator/src/token/attribute_value.rs @@ -58,12 +58,12 @@ impl AttributeValue { // No more required attributes, whitespace after attribute value, continue to next state return vec![ - Token::ElementOpen(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())), + Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes.clone())), Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes)) ]; } else if c == '>' { // End of opening tag - return vec![Token::ElementOpen(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))]; + return vec![Token::ElementOpenEnd(ElementOpenEnd::with_attributes(Arc::clone(&self.element), new_attributes))]; } else if c == '/' { // Beginning of self-closing tag return vec![Token::ElementSelfClose(ElementSelfClose::new_with_attributes(Arc::clone(&self.element), new_attributes))]; @@ -211,7 +211,7 @@ mod attribute_value_tests { assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute"); match &next_tokens[0] { - Token::ElementOpen(_) => (), + Token::ElementOpenEnd(_) => (), _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), } @@ -242,7 +242,7 @@ mod attribute_value_tests { assert!(!next_tokens.is_empty(), "Should accept '>' after attribute"); match &next_tokens[0] { - Token::ElementOpen(element_open) => { + Token::ElementOpenEnd(element_open) => { // Verify that the id attribute is tracked assert!(element_open.attributes_set.contains(&"id".to_string()), "Attribute 'id' should be tracked"); diff --git a/lib/xml_schema_validator/src/token/element_open_name.rs b/lib/xml_schema_validator/src/token/element_open_name.rs index cd1757b..eb6bbcd 100644 --- a/lib/xml_schema_validator/src/token/element_open_name.rs +++ b/lib/xml_schema_validator/src/token/element_open_name.rs @@ -34,10 +34,12 @@ impl ElementOpenName { } } - // No attributes, continue to element open or self-close + // No attributes return vec![ - Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element))), - Token::ElementSelfClose(ElementSelfClose::new()) + Token::AttributeName(AttributeName::new(Arc::clone(&self.element))), + Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element))), + Token::ElementSelfClose(ElementSelfClose::new()), + Token::ElementOpenName(self), ]; } else { // Whitespace in the middle of an element name is invalid @@ -59,7 +61,7 @@ impl ElementOpenName { } } - return vec![Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element)))]; + return vec![Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&self.element)))]; } else { // '>' before full element name is invalid return vec![]; @@ -79,7 +81,7 @@ impl ElementOpenName { if self.element.name.starts_with(&new_buffer) { // Still building the element name - return vec![Token::ElementName(ElementOpenName { + return vec![Token::ElementOpenName(ElementOpenName { buffer: new_buffer, element: Arc::clone(&self.element), })]; @@ -166,7 +168,7 @@ mod element_open_name_tests { assert!(!next_tokens.is_empty(), "Should accept '>' after full element name"); match &next_tokens[0] { - Token::ElementOpen(_) => (), + Token::ElementOpenEnd(_) => (), _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), } } @@ -182,10 +184,23 @@ mod element_open_name_tests { let next_tokens = element_name.append(' '); assert!(!next_tokens.is_empty(), "Should accept space after full element name"); - match &next_tokens[0] { - Token::ElementOpen(_) => (), - _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), - } + assert!(next_tokens.len() == 4); + assert!(next_tokens.iter() + .filter(|token| matches!(token, Token::ElementOpenName(_))) + .next() + .is_some()); + assert!(next_tokens.iter() + .filter(|token| matches!(token, Token::AttributeName(_))) + .next() + .is_some()); + assert!(next_tokens.iter() + .filter(|token| matches!(token, Token::ElementOpenEnd(_))) + .next() + .is_some()); + assert!(next_tokens.iter() + .filter(|token| matches!(token, Token::ElementSelfClose(_))) + .next() + .is_some()); } #[test] diff --git a/lib/xml_schema_validator/src/token/element_open_start.rs b/lib/xml_schema_validator/src/token/element_open_start.rs index 683fed1..cc0d9d7 100644 --- a/lib/xml_schema_validator/src/token/element_open_start.rs +++ b/lib/xml_schema_validator/src/token/element_open_start.rs @@ -17,7 +17,7 @@ impl ElementOpenStart { pub fn append(self, c: char) -> Vec { let element_name = ElementOpenName::new(Arc::clone(&self.element), c.to_string()); if let Ok(element_name) = element_name { - vec![Token::ElementName(element_name)] + vec![Token::ElementOpenName(element_name)] } else { vec![] } diff --git a/lib/xml_schema_validator/src/token/mod.rs b/lib/xml_schema_validator/src/token/mod.rs index a205cdc..af22365 100644 --- a/lib/xml_schema_validator/src/token/mod.rs +++ b/lib/xml_schema_validator/src/token/mod.rs @@ -27,9 +27,9 @@ pub use element_self_close::ElementSelfClose; pub enum Token { ElementCloseName(ElementCloseName), ElementCloseStart(ElementCloseStart), - ElementName(ElementOpenName), - ElementOpen(ElementOpenEnd), - ElementStart(ElementOpenStart), + ElementOpenName(ElementOpenName), + ElementOpenEnd(ElementOpenEnd), + ElementOpenStart(ElementOpenStart), EndOfFile(EndOfFile), StartOfFile(StartOfFile), TextContent(TextContent), @@ -43,9 +43,9 @@ impl Token { match self { Token::ElementCloseName(element_close_name) => element_close_name.append(c), Token::ElementCloseStart(element_close_start) => element_close_start.append(c), - Token::ElementName(element_name) => element_name.append(c), - Token::ElementOpen(element_open) => element_open.append(c), - Token::ElementStart(element_start) => element_start.append(c), + Token::ElementOpenName(element_name) => element_name.append(c), + Token::ElementOpenEnd(element_open) => element_open.append(c), + Token::ElementOpenStart(element_start) => element_start.append(c), Token::EndOfFile(end_of_file) => end_of_file.append(c), Token::StartOfFile(start_of_file) => start_of_file.append(c), Token::TextContent(text_content) => text_content.append(c), @@ -100,7 +100,7 @@ mod tests { assert_eq!(next_tokens.len(), 1, "Should have one transition"); match &next_tokens[0] { - Token::ElementStart(_) => (), + Token::ElementOpenStart(_) => (), _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]), } @@ -113,19 +113,19 @@ mod tests { #[test] fn test_element_start_transition() { let element = create_test_element(); - let token = Token::ElementStart(ElementOpenStart::new(Arc::clone(&element))); + let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element))); // Should accept first char of element name let next_tokens = token.append('r'); assert_eq!(next_tokens.len(), 1, "Should have one transition"); match &next_tokens[0] { - Token::ElementName(_) => (), + Token::ElementOpenName(_) => (), _ => panic!("Expected ElementName, got {:?}", next_tokens[0]), } // Should reject invalid char - let token = Token::ElementStart(ElementOpenStart::new(Arc::clone(&element))); + let token = Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element))); let next_tokens = token.append('x'); assert_eq!(next_tokens.len(), 0, "Should reject 'x'"); } @@ -133,20 +133,20 @@ mod tests { #[test] fn test_element_name_transition() { let element = create_test_element(); - let token = Token::ElementName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap()); + let token = Token::ElementOpenName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap()); // Continue building name let next_tokens = token.append('e'); assert_eq!(next_tokens.len(), 1, "Should have one transition"); match &next_tokens[0] { - Token::ElementName(_) => (), + Token::ElementOpenName(_) => (), _ => panic!("Expected ElementName, got {:?}", next_tokens[0]), } // Build complete name and end with > // Remove the unused buffer - let mut current_tokens = vec![Token::ElementStart(ElementOpenStart::new(Arc::clone(&element)))]; + let mut current_tokens = vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&element)))]; for c in "reasoning>".chars() { let mut new_tokens = Vec::new(); @@ -162,7 +162,7 @@ mod tests { // Check final state assert_eq!(current_tokens.len(), 1, "Should have one final transition"); match ¤t_tokens[0] { - Token::ElementOpen(_) => (), + Token::ElementOpenEnd(_) => (), _ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]), } } @@ -170,7 +170,7 @@ mod tests { #[test] fn test_element_open_end_transition() { let element = create_test_element(); - let token = Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&element))); + let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element))); // Should accept text content in mixed element let next_tokens = token.append('t'); @@ -182,7 +182,7 @@ mod tests { } // Should accept closing tag start - let token = Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&element))); + let token = Token::ElementOpenEnd(ElementOpenEnd::new(Arc::clone(&element))); let next_tokens = token.append('<'); assert_eq!(next_tokens.len(), 1, "Should accept '<'"); diff --git a/lib/xml_schema_validator/src/token/start_of_file.rs b/lib/xml_schema_validator/src/token/start_of_file.rs index db47fd9..582b2f5 100644 --- a/lib/xml_schema_validator/src/token/start_of_file.rs +++ b/lib/xml_schema_validator/src/token/start_of_file.rs @@ -14,7 +14,7 @@ impl StartOfFile { pub fn append(self, c: char) -> Vec { if '<' == c { - vec![Token::ElementStart(ElementOpenStart::new(Arc::clone(&self.element)))] + vec![Token::ElementOpenStart(ElementOpenStart::new(Arc::clone(&self.element)))] } else { vec![] } @@ -58,7 +58,7 @@ mod start_of_file_tests { assert!(!next_tokens.is_empty(), "Should accept '<'"); match &next_tokens[0] { - Token::ElementStart(_) => (), + Token::ElementOpenStart(_) => (), _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]), } } diff --git a/qwq_tokenizer_config.json b/qwq_tokenizer_config.json index 35c4b18..58712dd 100644 --- a/qwq_tokenizer_config.json +++ b/qwq_tokenizer_config.json @@ -227,7 +227,7 @@ "<|video_pad|>" ], "bos_token": null, - "chat_template": "{%- for message in messages %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + "chat_template": "{%- for message in messages %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content }}\n {%- if not loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '\\n<|im_start|>assistant\\n' }}\n{%- endif %}", "clean_up_tokenization_spaces": false, "eos_token": "<|im_end|>", "errors": "replace", diff --git a/scripts/deploy.sh b/scripts/deploy.sh index f361401..1f20c37 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -45,6 +45,7 @@ output=$(runpodctl create pod \ --env "SIA_HF_API_KEY=$SIA_HF_API_KEY" \ --env "SIA_QWQ_ENABLED=1" \ --env "JUPYTER_PASSWORD=1" \ + --communityCloud ) echo "$output"