llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed token cache for whitespace bug

Browse Source
This commit is contained in:
Geens
2025-04-13 10:03:14 +02:00
parent ff82173f2a
commit f4de9e7571
1 changed files with 11 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
lib/xml_schema_validator/python/xml_schema_validator/__init__.py
View File

@@ -34,25 +34,21 @@ class XmlLogitsProcessor(LogitsProcessor):
# Create a mapping from token id to decoded text
# This ensures we capture whitespace correctly
items = {}
for token, id in vocab.items():
# Properly decode each token to get its actual string representation
# This preserves whitespace characters
if isinstance(token, str):
# For string tokens, use them directly
items[id] = token
else:
# For byte tokens, ensure they're decoded properly
items[id] = tokenizer.convert_ids_to_tokens([id])[0]
#for token, id in vocab.items():
# # Properly decode each token to get its actual string representation
# # This preserves whitespace characters
# if isinstance(token, str):
# # For string tokens, use them directly
# items[id] = token
# else:
# print("b")
# # For byte tokens, ensure they're decoded properly
# items[id] = tokenizer.convert_ids_to_tokens([id])[0]
# Additional special handling for whitespace tokens
# Some tokenizers have special tokens for whitespace
for id in range(len(vocab)):
if id in items:
# Try decoding single tokens to find whitespace
decoded = tokenizer.decode([id])
if decoded and (decoded.isspace() or decoded.startswith(' ')):
# This token represents whitespace - ensure it's preserved
items[id] = decoded
items[id] = tokenizer.decode([id])
self.core = XmlLogitsProcessorCore(items, schema_text)
else: