From f4de9e75718dc8e0ba325a0a999596da99d83d1f Mon Sep 17 00:00:00 2001 From: Geens Date: Sun, 13 Apr 2025 10:03:14 +0200 Subject: [PATCH] Fixed token cache for whitespace bug --- .../python/xml_schema_validator/__init__.py | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/xml_schema_validator/python/xml_schema_validator/__init__.py b/lib/xml_schema_validator/python/xml_schema_validator/__init__.py index e30e3bc..2a10f08 100644 --- a/lib/xml_schema_validator/python/xml_schema_validator/__init__.py +++ b/lib/xml_schema_validator/python/xml_schema_validator/__init__.py @@ -34,25 +34,21 @@ class XmlLogitsProcessor(LogitsProcessor): # Create a mapping from token id to decoded text # This ensures we capture whitespace correctly items = {} - for token, id in vocab.items(): - # Properly decode each token to get its actual string representation - # This preserves whitespace characters - if isinstance(token, str): - # For string tokens, use them directly - items[id] = token - else: - # For byte tokens, ensure they're decoded properly - items[id] = tokenizer.convert_ids_to_tokens([id])[0] + #for token, id in vocab.items(): + # # Properly decode each token to get its actual string representation + # # This preserves whitespace characters + # if isinstance(token, str): + # # For string tokens, use them directly + # items[id] = token + # else: + # print("b") + # # For byte tokens, ensure they're decoded properly + # items[id] = tokenizer.convert_ids_to_tokens([id])[0] # Additional special handling for whitespace tokens # Some tokenizers have special tokens for whitespace for id in range(len(vocab)): - if id in items: - # Try decoding single tokens to find whitespace - decoded = tokenizer.decode([id]) - if decoded and (decoded.isspace() or decoded.startswith(' ')): - # This token represents whitespace - ensure it's preserved - items[id] = decoded + items[id] = tokenizer.decode([id]) self.core = XmlLogitsProcessorCore(items, schema_text) else: