diff --git a/iterations/iteration_20250409_151814_463.xml b/iterations/iteration_20250409_151814_463.xml new file mode 100644 index 0000000..218a06d --- /dev/null +++ b/iterations/iteration_20250409_151814_463.xml @@ -0,0 +1,16 @@ + +<context + context="45.92%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="59403497472" + stdin="0" + time="2025-04-09T15:06:29Z"> +</context><reasoning> +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +</reasoning> \ No newline at end of file diff --git a/iterations/iteration_20250409_151841_584.xml b/iterations/iteration_20250409_151841_584.xml new file mode 100644 index 0000000..eb609e4 --- /dev/null +++ b/iterations/iteration_20250409_151841_584.xml @@ -0,0 +1,21 @@ + +<context + context="53.42%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="60000591872" + stdin="0" + time="2025-04-09T15:18:14Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> +</context><write_stdout> +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +</write_stdout> \ No newline at end of file diff --git a/iterations/iteration_20250409_152126_231.xml b/iterations/iteration_20250409_152126_231.xml new file mode 100644 index 0000000..fbb6f34 --- /dev/null +++ b/iterations/iteration_20250409_152126_231.xml @@ -0,0 +1,40 @@ + +<context + context="55.18%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="60161978368" + stdin="0" + time="2025-04-09T15:18:41Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> +</context><reasoning> +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +</reasoning> \ No newline at end of file diff --git a/iterations/iteration_20250409_152213_900.xml b/iterations/iteration_20250409_152213_900.xml new file mode 100644 index 0000000..457ccdd --- /dev/null +++ b/iterations/iteration_20250409_152213_900.xml @@ -0,0 +1,47 @@ + +<context + context="74.29%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="59503431680" + stdin="61" + time="2025-04-09T15:21:26Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> + <reasoning id="20250409_152126_231"> + <![CDATA[ +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +]]> + </reasoning> +</context><reasoning> +Alright, let me start by looking at the current context. The user hasn't provided any input yet since the last message I sent. My goal is to prompt them for their task, so I should make sure they know I'm waiting. The previous action was a write_stdout asking for their task. Now, since there's no new input (stdin is 0), I need to decide what to do next. + +The instructions say to avoid spamming the user, so sending another message right away might not be best. Instead, maybe I should set up a repeat script to periodically check for input or remind them if needed. But the user might prefer a single prompt. Alternatively, a reasoning note can log that I'm waiting. Since I must choose one action, a reasoning entry makes sense here to document the status without further action. That way, I stay within the rules and wait for the user's next input without unnecessary steps. +</reasoning> \ No newline at end of file diff --git a/iterations/iteration_20250409_152308_303.xml b/iterations/iteration_20250409_152308_303.xml new file mode 100644 index 0000000..63c21a3 --- /dev/null +++ b/iterations/iteration_20250409_152308_303.xml @@ -0,0 +1,50 @@ + +<context + context="79.76%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="59883008000" + stdin="61" + time="2025-04-09T15:22:13Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> + <reasoning id="20250409_152126_231"> + <![CDATA[ +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +]]> + </reasoning> + <reasoning id="20250409_152213_900"> + <![CDATA[ +Alright, let me start by looking at the current context. The user hasn't provided any input yet since the last message I sent. My goal is to prompt them for their task, so I should make sure they know I'm waiting. The previous action was a write_stdout asking for their task. Now, since there's no new input (stdin is 0), I need to decide what to do next. + +The instructions say to avoid spamming the user, so sending another message right away might not be best. Instead, maybe I should set up a repeat script to periodically check for input or remind them if needed. But the user might prefer a single prompt. Alternatively, a reasoning note can log that I'm waiting. Since I must choose one action, a reasoning entry makes sense here to document the status without further action. That way, I stay within the rules and wait for the user's next input without unnecessary steps. +]]> + </reasoning> +</context><read_stdin/> \ No newline at end of file diff --git a/iterations/iteration_20250409_152354_453.xml b/iterations/iteration_20250409_152354_453.xml new file mode 100644 index 0000000..e8c441c --- /dev/null +++ b/iterations/iteration_20250409_152354_453.xml @@ -0,0 +1,55 @@ + +<context + context="81.15%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="59900334080" + stdin="0" + time="2025-04-09T15:23:08Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> + <reasoning id="20250409_152126_231"> + <![CDATA[ +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +]]> + </reasoning> + <reasoning id="20250409_152213_900"> + <![CDATA[ +Alright, let me start by looking at the current context. The user hasn't provided any input yet since the last message I sent. My goal is to prompt them for their task, so I should make sure they know I'm waiting. The previous action was a write_stdout asking for their task. Now, since there's no new input (stdin is 0), I need to decide what to do next. + +The instructions say to avoid spamming the user, so sending another message right away might not be best. Instead, maybe I should set up a repeat script to periodically check for input or remind them if needed. But the user might prefer a single prompt. Alternatively, a reasoning note can log that I'm waiting. Since I must choose one action, a reasoning entry makes sense here to document the status without further action. That way, I stay within the rules and wait for the user's next input without unnecessary steps. +]]> + </reasoning> + <read_stdin id="20250409_152308_303"> + <![CDATA[Hi, I'm Niels. Can you check the /root/sia/procedures folder?]]> + </read_stdin> +</context><reasoning> +Okay, the user, Niels, has responded. They want me to check the /root/sia/procedures folder. First, I need to verify if that directory exists. I'll run a single script to check the existence of the folder and list its contents if it's there. That way, I can provide feedback based on what's found. If the folder doesn't exist, I'll inform the user. If it does, I'll list the files and maybe ask for further instructions. Need to make sure the command is correct, using 'ls' with appropriate options. Also, since it's a single action, I should structure the XML properly. +</reasoning> \ No newline at end of file diff --git a/iterations/iteration_20250409_152415_041.xml b/iterations/iteration_20250409_152415_041.xml new file mode 100644 index 0000000..f23616f --- /dev/null +++ b/iterations/iteration_20250409_152415_041.xml @@ -0,0 +1,64 @@ + +<context + context="85.3%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="59974135808" + stdin="0" + time="2025-04-09T15:23:54Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> + <reasoning id="20250409_152126_231"> + <![CDATA[ +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +]]> + </reasoning> + <reasoning id="20250409_152213_900"> + <![CDATA[ +Alright, let me start by looking at the current context. The user hasn't provided any input yet since the last message I sent. My goal is to prompt them for their task, so I should make sure they know I'm waiting. The previous action was a write_stdout asking for their task. Now, since there's no new input (stdin is 0), I need to decide what to do next. + +The instructions say to avoid spamming the user, so sending another message right away might not be best. Instead, maybe I should set up a repeat script to periodically check for input or remind them if needed. But the user might prefer a single prompt. Alternatively, a reasoning note can log that I'm waiting. Since I must choose one action, a reasoning entry makes sense here to document the status without further action. That way, I stay within the rules and wait for the user's next input without unnecessary steps. +]]> + </reasoning> + <read_stdin id="20250409_152308_303"> + <![CDATA[Hi, I'm Niels. Can you check the /root/sia/procedures folder?]]> + </read_stdin> + <reasoning id="20250409_152354_453"> + <![CDATA[ +Okay, the user, Niels, has responded. They want me to check the /root/sia/procedures folder. First, I need to verify if that directory exists. I'll run a single script to check the existence of the folder and list its contents if it's there. That way, I can provide feedback based on what's found. If the folder doesn't exist, I'll inform the user. If it does, I'll list the files and maybe ask for further instructions. Need to make sure the command is correct, using 'ls' with appropriate options. Also, since it's a single action, I should structure the XML properly. +]]> + </reasoning> +</context><single> + if [ -d "/root/sia/procedures" ]; then + ls -l /root/sia/procedures + else + echo "Directory /root/sia/procedures does not exist." + fi +</single> \ No newline at end of file diff --git a/iterations/iteration_20250409_152518_309.xml b/iterations/iteration_20250409_152518_309.xml new file mode 100644 index 0000000..de314f0 --- /dev/null +++ b/iterations/iteration_20250409_152518_309.xml @@ -0,0 +1,87 @@ + +<context + context="92.68%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="59848675328" + stdin="0" + time="2025-04-09T15:24:15Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> + <reasoning id="20250409_152126_231"> + <![CDATA[ +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +]]> + </reasoning> + <reasoning id="20250409_152213_900"> + <![CDATA[ +Alright, let me start by looking at the current context. The user hasn't provided any input yet since the last message I sent. My goal is to prompt them for their task, so I should make sure they know I'm waiting. The previous action was a write_stdout asking for their task. Now, since there's no new input (stdin is 0), I need to decide what to do next. + +The instructions say to avoid spamming the user, so sending another message right away might not be best. Instead, maybe I should set up a repeat script to periodically check for input or remind them if needed. But the user might prefer a single prompt. Alternatively, a reasoning note can log that I'm waiting. Since I must choose one action, a reasoning entry makes sense here to document the status without further action. That way, I stay within the rules and wait for the user's next input without unnecessary steps. +]]> + </reasoning> + <read_stdin id="20250409_152308_303"> + <![CDATA[Hi, I'm Niels. Can you check the /root/sia/procedures folder?]]> + </read_stdin> + <reasoning id="20250409_152354_453"> + <![CDATA[ +Okay, the user, Niels, has responded. They want me to check the /root/sia/procedures folder. First, I need to verify if that directory exists. I'll run a single script to check the existence of the folder and list its contents if it's there. That way, I can provide feedback based on what's found. If the folder doesn't exist, I'll inform the user. If it does, I'll list the files and maybe ask for further instructions. Need to make sure the command is correct, using 'ls' with appropriate options. Also, since it's a single action, I should structure the XML properly. +]]> + </reasoning> + <single exit_code="0" id="20250409_152415_041"> + <![CDATA[ + if [ -d "/root/sia/procedures" ]; then + ls -l /root/sia/procedures + else + echo "Directory /root/sia/procedures does not exist." + fi +]]> + <stdout> + <![CDATA[total 4 +-rw-r--r-- 1 root root 3230 Apr 9 14:50 README.md +drwxr-xr-x 2 root root 26 Apr 9 14:50 filesystem_design +drwxr-xr-x 2 root root 26 Apr 9 14:50 self_improvement +drwxr-xr-x 2 root root 34 Apr 9 14:50 tool_management +drwxr-xr-x 2 root root 84 Apr 9 14:50 user_communication +drwxr-xr-x 2 root root 58 Apr 9 14:50 using_procedures +drwxr-xr-x 2 root root 34 Apr 9 14:50 version_control +]]> + </stdout> + <stderr/> + </single> +</context><reasoning> +Okay, the user, Niels, asked me to check the /root/sia/procedures folder. I ran a script to see if the directory exists and list its contents. The output shows that the directory exists and contains several subdirectories and a README.md file. + +First, I should inform Niels that the directory exists and provide the list of contents. The output includes the README, which might have important information. The subdirectories like filesystem_design, self_improvement, etc., suggest organized procedures. + +Next, I should ask Niels what specific action they want me to take next. Maybe they want me to review the contents, execute a procedure, or look into a particular subdirectory. Since the user's request was to "check" the folder, the initial step is done. Now, I need to prompt for further instructions to proceed appropriately. + +I should make sure the response is clear and asks for next steps. Also, check if there's any error in the script output, but the exit code was 0, so it ran successfully. All set to send the response. +</reasoning> \ No newline at end of file diff --git a/iterations/iteration_20250409_152605_559.xml b/iterations/iteration_20250409_152605_559.xml new file mode 100644 index 0000000..de0e8d0 --- /dev/null +++ b/iterations/iteration_20250409_152605_559.xml @@ -0,0 +1,94 @@ + +<context + context="98.97%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="60295892992" + stdin="0" + time="2025-04-09T15:25:18Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> + <reasoning id="20250409_152126_231"> + <![CDATA[ +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +]]> + </reasoning> + <reasoning id="20250409_152213_900"> + <![CDATA[ +Alright, let me start by looking at the current context. The user hasn't provided any input yet since the last message I sent. My goal is to prompt them for their task, so I should make sure they know I'm waiting. The previous action was a write_stdout asking for their task. Now, since there's no new input (stdin is 0), I need to decide what to do next. + +The instructions say to avoid spamming the user, so sending another message right away might not be best. Instead, maybe I should set up a repeat script to periodically check for input or remind them if needed. But the user might prefer a single prompt. Alternatively, a reasoning note can log that I'm waiting. Since I must choose one action, a reasoning entry makes sense here to document the status without further action. That way, I stay within the rules and wait for the user's next input without unnecessary steps. +]]> + </reasoning> + <read_stdin id="20250409_152308_303"> + <![CDATA[Hi, I'm Niels. Can you check the /root/sia/procedures folder?]]> + </read_stdin> + <reasoning id="20250409_152354_453"> + <![CDATA[ +Okay, the user, Niels, has responded. They want me to check the /root/sia/procedures folder. First, I need to verify if that directory exists. I'll run a single script to check the existence of the folder and list its contents if it's there. That way, I can provide feedback based on what's found. If the folder doesn't exist, I'll inform the user. If it does, I'll list the files and maybe ask for further instructions. Need to make sure the command is correct, using 'ls' with appropriate options. Also, since it's a single action, I should structure the XML properly. +]]> + </reasoning> + <single exit_code="0" id="20250409_152415_041"> + <![CDATA[ + if [ -d "/root/sia/procedures" ]; then + ls -l /root/sia/procedures + else + echo "Directory /root/sia/procedures does not exist." + fi +]]> + <stdout> + <![CDATA[total 4 +-rw-r--r-- 1 root root 3230 Apr 9 14:50 README.md +drwxr-xr-x 2 root root 26 Apr 9 14:50 filesystem_design +drwxr-xr-x 2 root root 26 Apr 9 14:50 self_improvement +drwxr-xr-x 2 root root 34 Apr 9 14:50 tool_management +drwxr-xr-x 2 root root 84 Apr 9 14:50 user_communication +drwxr-xr-x 2 root root 58 Apr 9 14:50 using_procedures +drwxr-xr-x 2 root root 34 Apr 9 14:50 version_control +]]> + </stdout> + <stderr/> + </single> + <reasoning id="20250409_152518_309"> + <![CDATA[ +Okay, the user, Niels, asked me to check the /root/sia/procedures folder. I ran a script to see if the directory exists and list its contents. The output shows that the directory exists and contains several subdirectories and a README.md file. + +First, I should inform Niels that the directory exists and provide the list of contents. The output includes the README, which might have important information. The subdirectories like filesystem_design, self_improvement, etc., suggest organized procedures. + +Next, I should ask Niels what specific action they want me to take next. Maybe they want me to review the contents, execute a procedure, or look into a particular subdirectory. Since the user's request was to "check" the folder, the initial step is done. Now, I need to prompt for further instructions to proceed appropriately. + +I should make sure the response is clear and asks for next steps. Also, check if there's any error in the script output, but the exit code was 0, so it ran successfully. All set to send the response. +]]> + </reasoning> +</context><reasoning> +Okay, the user asked me to check the /root/sia/procedures folder. I ran the script and found that the directory exists with several subdirectories and a README.md. Now I need to inform them of the results and ask for next steps. Let me structure the response clearly. + +First, confirm the directory exists. List the contents from the stdout. Mention the README and subdirectories. Then, ask if they want to explore a specific subdirectory or need more details. Keep it friendly and helpful. +</reasoning> \ No newline at end of file diff --git a/iterations/iteration_20250409_152657_014.xml b/iterations/iteration_20250409_152657_014.xml new file mode 100644 index 0000000..b7e74cb --- /dev/null +++ b/iterations/iteration_20250409_152657_014.xml @@ -0,0 +1,109 @@ + +<context + context="102.37%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="60269191168" + stdin="0" + time="2025-04-09T15:26:05Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> + <reasoning id="20250409_152126_231"> + <![CDATA[ +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +]]> + </reasoning> + <reasoning id="20250409_152213_900"> + <![CDATA[ +Alright, let me start by looking at the current context. The user hasn't provided any input yet since the last message I sent. My goal is to prompt them for their task, so I should make sure they know I'm waiting. The previous action was a write_stdout asking for their task. Now, since there's no new input (stdin is 0), I need to decide what to do next. + +The instructions say to avoid spamming the user, so sending another message right away might not be best. Instead, maybe I should set up a repeat script to periodically check for input or remind them if needed. But the user might prefer a single prompt. Alternatively, a reasoning note can log that I'm waiting. Since I must choose one action, a reasoning entry makes sense here to document the status without further action. That way, I stay within the rules and wait for the user's next input without unnecessary steps. +]]> + </reasoning> + <read_stdin id="20250409_152308_303"> + <![CDATA[Hi, I'm Niels. Can you check the /root/sia/procedures folder?]]> + </read_stdin> + <reasoning id="20250409_152354_453"> + <![CDATA[ +Okay, the user, Niels, has responded. They want me to check the /root/sia/procedures folder. First, I need to verify if that directory exists. I'll run a single script to check the existence of the folder and list its contents if it's there. That way, I can provide feedback based on what's found. If the folder doesn't exist, I'll inform the user. If it does, I'll list the files and maybe ask for further instructions. Need to make sure the command is correct, using 'ls' with appropriate options. Also, since it's a single action, I should structure the XML properly. +]]> + </reasoning> + <single exit_code="0" id="20250409_152415_041"> + <![CDATA[ + if [ -d "/root/sia/procedures" ]; then + ls -l /root/sia/procedures + else + echo "Directory /root/sia/procedures does not exist." + fi +]]> + <stdout> + <![CDATA[total 4 +-rw-r--r-- 1 root root 3230 Apr 9 14:50 README.md +drwxr-xr-x 2 root root 26 Apr 9 14:50 filesystem_design +drwxr-xr-x 2 root root 26 Apr 9 14:50 self_improvement +drwxr-xr-x 2 root root 34 Apr 9 14:50 tool_management +drwxr-xr-x 2 root root 84 Apr 9 14:50 user_communication +drwxr-xr-x 2 root root 58 Apr 9 14:50 using_procedures +drwxr-xr-x 2 root root 34 Apr 9 14:50 version_control +]]> + </stdout> + <stderr/> + </single> + <reasoning id="20250409_152518_309"> + <![CDATA[ +Okay, the user, Niels, asked me to check the /root/sia/procedures folder. I ran a script to see if the directory exists and list its contents. The output shows that the directory exists and contains several subdirectories and a README.md file. + +First, I should inform Niels that the directory exists and provide the list of contents. The output includes the README, which might have important information. The subdirectories like filesystem_design, self_improvement, etc., suggest organized procedures. + +Next, I should ask Niels what specific action they want me to take next. Maybe they want me to review the contents, execute a procedure, or look into a particular subdirectory. Since the user's request was to "check" the folder, the initial step is done. Now, I need to prompt for further instructions to proceed appropriately. + +I should make sure the response is clear and asks for next steps. Also, check if there's any error in the script output, but the exit code was 0, so it ran successfully. All set to send the response. +]]> + </reasoning> + <reasoning id="20250409_152605_559"> + <![CDATA[ +Okay, the user asked me to check the /root/sia/procedures folder. I ran the script and found that the directory exists with several subdirectories and a README.md. Now I need to inform them of the results and ask for next steps. Let me structure the response clearly. + +First, confirm the directory exists. List the contents from the stdout. Mention the README and subdirectories. Then, ask if they want to explore a specific subdirectory or need more details. Keep it friendly and helpful. +]]> + </reasoning> +</context><write_stdout> +The directory /root/sia/procedures exists. Its contents are: + +- README.md (3230 bytes) +- filesystem_design/ +- self_improvement/ +- tool_management/ +- user_communication/ +- using_procedures/ +- version_control/ + +Would you like me to examine any specific subdirectory or file within this folder first? +</write_stdout> \ No newline at end of file diff --git a/iterations/iteration_20250409_153052_172.xml b/iterations/iteration_20250409_153052_172.xml new file mode 100644 index 0000000..5ce89a9 --- /dev/null +++ b/iterations/iteration_20250409_153052_172.xml @@ -0,0 +1,120 @@ + +<context + context="104.98%" + disk_total="214748364800" + disk_used="113238601728" + memory_total="540877729792" + memory_used="59822100480" + stdin="90" + time="2025-04-09T15:29:16Z"> + <reasoning id="20250409_151814_463"> + <![CDATA[ +Okay, I need to start working on the user's task. Let me look at the context provided. The context usage is at 45.92%, which is manageable. The user hasn't provided any specific instructions yet, so I should probably ask them what they need help with. But first, I should check if there's any existing context or files I need to consider. Since the context seems empty, maybe I should prompt the user for their request. + +Wait, the user's initial message might just be setting up the environment. Let me make sure I'm following the correct流程. The user provided the XML schema and context, so my first step is to respond with an action. Since there's no user input yet, I should probably use a write_stdout action to ask them for their task. That way, they can provide the details I need to proceed. + +Alternatively, maybe I should check the filesystem or set up some initial scripts. But without knowing the user's goal, it's better to ask first. Let me confirm by looking at the user interaction section in the instructions. It says to be a helpful assistant and communicate via standard I/O. So yes, sending a write_stdout message is the right move here. I'll craft a friendly message asking the user to specify their task or question. That should get things started. +]]> + </reasoning> + <write_stdout id="20250409_151841_584"> + <![CDATA[ +Hello! I'm SIA, the Self Improving Agent. How can I assist you today? Please provide your task or question, and I'll get started on it. +]]> + </write_stdout> + <reasoning id="20250409_152126_231"> + <![CDATA[ +Okay, let me process the user's setup. The initial context shows that SIA has just started and sent a greeting to the user. The next step is to wait for the user's response. Since the user hasn't provided any input yet (stdin is 0), SIA should continue monitoring for input and possibly prompt again if needed. However, according to the problem statement, the user's next input is the context provided, which includes SIA's initial actions. + +Now, in this iteration, the context includes the greeting message sent to the user. The user (in this scenario) is likely expecting SIA to proceed based on their input. But since the user's actual task isn't specified yet, SIA should probably check if there's any new input. Since stdin is still 0, maybe there's nothing new. Alternatively, perhaps SIA should set up a repeat script to continuously check for new input or prompt the user again if necessary. + +Wait, the context's stdin attribute is 0, meaning there's no incoming data. So SIA should probably just wait, but since it's an iteration, maybe it's better to add a repeat script to check for new input periodically. Alternatively, maybe the user expects SIA to proceed with some default action, but the instructions say to wait for user tasks. + +Alternatively, maybe the user's next step is to give a task, but in the provided context, it's the first interaction. Since the user's message after the initial setup is just the context, perhaps this is the first iteration after the greeting. The correct action here is to continue waiting, but ensure that SIA is ready to process when the user responds. + +Since the user hasn't provided a task yet, SIA should not take any action except maybe a repeat script to monitor or a reasoning step to confirm the next steps. However, according to the problem's current context, the next step is to respond with the next action. Since the previous action was a write_stdout, and there's no new input, perhaps the best move is to do nothing except maybe a reasoning note to confirm the status. But the system requires exactly one action. + +Wait, the user's initial input was the XML schema and the context example. The assistant's first response was the greeting. Now, the next iteration (the current one) has the context with that greeting. Since the user hasn't input anything yet, SIA should probably just issue a reasoning action to note that it's waiting for the user's task. Alternatively, maybe a repeat script to check stdin again. But the system requires one action. Since there's no new input, perhaps the best action is to issue another reasoning step to explain that it's waiting, or perhaps a read_stdin to check if there's any input. But read_stdin should only be used if there's data available. Since stdin is 0, doing read_stdin would result in an empty read, but maybe that's allowed. + +Alternatively, maybe the user expects SIA to proceed with the next step, but without a task, it's stuck. Therefore, the correct action is to repeat the prompt or ensure that the user knows to provide a task. However, the initial write_stdout is already there. To avoid spamming, maybe just a reasoning note. + +Hmm. The key is to follow the XML schema and respond with exactly one action. Since the previous iteration already sent the greeting, in this iteration, perhaps SIA should do a reasoning action to note that it's waiting for the user's input. That's acceptable. So the next action would be a reasoning element explaining it's awaiting further instructions. +]]> + </reasoning> + <reasoning id="20250409_152213_900"> + <![CDATA[ +Alright, let me start by looking at the current context. The user hasn't provided any input yet since the last message I sent. My goal is to prompt them for their task, so I should make sure they know I'm waiting. The previous action was a write_stdout asking for their task. Now, since there's no new input (stdin is 0), I need to decide what to do next. + +The instructions say to avoid spamming the user, so sending another message right away might not be best. Instead, maybe I should set up a repeat script to periodically check for input or remind them if needed. But the user might prefer a single prompt. Alternatively, a reasoning note can log that I'm waiting. Since I must choose one action, a reasoning entry makes sense here to document the status without further action. That way, I stay within the rules and wait for the user's next input without unnecessary steps. +]]> + </reasoning> + <read_stdin id="20250409_152308_303"> + <![CDATA[Hi, I'm Niels. Can you check the /root/sia/procedures folder?]]> + </read_stdin> + <reasoning id="20250409_152354_453"> + <![CDATA[ +Okay, the user, Niels, has responded. They want me to check the /root/sia/procedures folder. First, I need to verify if that directory exists. I'll run a single script to check the existence of the folder and list its contents if it's there. That way, I can provide feedback based on what's found. If the folder doesn't exist, I'll inform the user. If it does, I'll list the files and maybe ask for further instructions. Need to make sure the command is correct, using 'ls' with appropriate options. Also, since it's a single action, I should structure the XML properly. +]]> + </reasoning> + <single exit_code="0" id="20250409_152415_041"> + <![CDATA[ + if [ -d "/root/sia/procedures" ]; then + ls -l /root/sia/procedures + else + echo "Directory /root/sia/procedures does not exist." + fi +]]> + <stdout> + <![CDATA[total 4 +-rw-r--r-- 1 root root 3230 Apr 9 14:50 README.md +drwxr-xr-x 2 root root 26 Apr 9 14:50 filesystem_design +drwxr-xr-x 2 root root 26 Apr 9 14:50 self_improvement +drwxr-xr-x 2 root root 34 Apr 9 14:50 tool_management +drwxr-xr-x 2 root root 84 Apr 9 14:50 user_communication +drwxr-xr-x 2 root root 58 Apr 9 14:50 using_procedures +drwxr-xr-x 2 root root 34 Apr 9 14:50 version_control +]]> + </stdout> + <stderr/> + </single> + <reasoning id="20250409_152518_309"> + <![CDATA[ +Okay, the user, Niels, asked me to check the /root/sia/procedures folder. I ran a script to see if the directory exists and list its contents. The output shows that the directory exists and contains several subdirectories and a README.md file. + +First, I should inform Niels that the directory exists and provide the list of contents. The output includes the README, which might have important information. The subdirectories like filesystem_design, self_improvement, etc., suggest organized procedures. + +Next, I should ask Niels what specific action they want me to take next. Maybe they want me to review the contents, execute a procedure, or look into a particular subdirectory. Since the user's request was to "check" the folder, the initial step is done. Now, I need to prompt for further instructions to proceed appropriately. + +I should make sure the response is clear and asks for next steps. Also, check if there's any error in the script output, but the exit code was 0, so it ran successfully. All set to send the response. +]]> + </reasoning> + <reasoning id="20250409_152605_559"> + <![CDATA[ +Okay, the user asked me to check the /root/sia/procedures folder. I ran the script and found that the directory exists with several subdirectories and a README.md. Now I need to inform them of the results and ask for next steps. Let me structure the response clearly. + +First, confirm the directory exists. List the contents from the stdout. Mention the README and subdirectories. Then, ask if they want to explore a specific subdirectory or need more details. Keep it friendly and helpful. +]]> + </reasoning> + <write_stdout id="20250409_152657_014"> + <![CDATA[ +The directory /root/sia/procedures exists. Its contents are: + +- README.md (3230 bytes) +- filesystem_design/ +- self_improvement/ +- tool_management/ +- user_communication/ +- using_procedures/ +- version_control/ + +Would you like me to examine any specific subdirectory or file within this folder first? +]]> + </write_stdout> +</context><reasoning> +Okay, the user, Niels, wants me to check the /root/sia/procedures folder. I've already done that and listed the contents. Now he might want me to look into one of the subdirectories. Let me see his response. + +Wait, the last entry in the context is my write_stdout asking if he wants to examine a specific subdirectory. The next step depends on his answer. But in the current context, after my question, there's no new input yet. The stdin is at 90, which probably means there's some input waiting. Oh, right, there's a read_stdin entry with his initial message, but maybe he's sent another response now. + +Wait, looking at the provided context, the last entries are my question and then... Hmm, in the given context, after the write_stdout (id 20250409_152657_014), there's no further read_stdin. The previous read_stdin was his initial request. So perhaps I need to wait for his next input. But according to the problem, I must issue exactly one action now. Since there's no new input (assuming stdin is now 0 again), maybe I should prompt again or check if he wants to proceed. Alternatively, maybe he expects me to proceed with one of the subdirectories. But without instructions, I should wait. However, the user might have provided more input. Wait, in the current context, the stdin attribute is 90, which might indicate there's data. Oh, the initial context's stdin is 90, but in the context elements, the last read_stdin was his first message. Maybe there's more input. Wait, the context's stdin attribute is 90, which could mean there's 90 bytes available. So perhaps I should read that next. + +Wait, the context element has stdin="90", which suggests there's input waiting. So my next action should be to read_stdin to get the user's response to my previous question. That makes sense. So first, I need to read the stdin to get his answer. Let me check: the user's previous message was the initial request, and after my response, he might have answered. Since the current context hasn't processed that yet, I should first read_stdin to capture his next input. Then, based on that, proceed. So the next action should be a read_stdin. +</reasoning> \ No newline at end of file diff --git a/lib/xml_schema_validator/Cargo.toml b/lib/xml_schema_validator/Cargo.toml index 6ba8f0b..bc6b2b8 100644 --- a/lib/xml_schema_validator/Cargo.toml +++ b/lib/xml_schema_validator/Cargo.toml @@ -10,7 +10,8 @@ name = "xml_schema_validator" crate-type = ["cdylib", "rlib"] [dependencies] -pyo3 = { version = "0.24", features = ["extension-module"] } +pyo3 = { version = "0.24", features = ["extension-module", "auto-initialize"] } quick-xml = { version = "0.28.1", features = ["serde", "serialize"] } serde = { version = "1", features = ["derive"] } +serde_json = "1.0" thiserror = "1.0.38" \ No newline at end of file diff --git a/lib/xml_schema_validator/README.md b/lib/xml_schema_validator/README.md index 7698587..fad768e 100644 --- a/lib/xml_schema_validator/README.md +++ b/lib/xml_schema_validator/README.md @@ -1,3 +1,3 @@ # XML Schema Validator -XML Schema validation library optimized for LLM token validation. +An XML Schema validation library optimized for LLM token validation. This library allows you to validate XML tokens against an XSD schema, enabling constrained token generation in language models. \ No newline at end of file diff --git a/lib/xml_schema_validator/python/xml_schema_validator/__init__.py b/lib/xml_schema_validator/python/xml_schema_validator/__init__.py index 7bbc9e1..c0393a0 100644 --- a/lib/xml_schema_validator/python/xml_schema_validator/__init__.py +++ b/lib/xml_schema_validator/python/xml_schema_validator/__init__.py @@ -12,22 +12,31 @@ class XmlLogitsProcessor(LogitsProcessor): by setting their logits to negative infinity. """ - def __init__(self, tokenizer: AutoTokenizer, schema_text: str): + def __init__(self, tokenizer: AutoTokenizer, schema_text: str = None, core=None): """ Initialize the processor with a schema and tokenizer. Args: tokenizer: The tokenizer to use for decoding tokens schema_text: The XSD schema text to validate against + core: An existing core processor (for internal use in copy()) """ self.eos_token_id = tokenizer.eos_token_id self.tokenizer = tokenizer - vocab = tokenizer.get_vocab() # This is {token: id} - items = dict() - for token, id in vocab.items(): - items[id] = token - if schema_text: + + if core is not None: + # Used for copy() operation + self.core = core + elif schema_text: + # Normal initialization + vocab = tokenizer.get_vocab() # This is {token: id} + items = dict() + for token, id in vocab.items(): + items[id] = token self.core = XmlLogitsProcessorCore(items, schema_text) + else: + raise ValueError("Either schema_text or core must be provided") + self.prompt_length = None self.is_first_call = True @@ -60,7 +69,10 @@ class XmlLogitsProcessor(LogitsProcessor): batch_processor = self.core.copy() if generated_text: - batch_processor.append(generated_text) + # Try to append the current text to the validator + append_result = batch_processor.append(generated_text) + + # If we've reached a valid EOF state, only allow the EOS token if batch_processor.eof(): vocab_size = scores.shape[-1] valid_tokens_mask = torch.zeros(vocab_size, dtype=torch.bool, device=scores.device) @@ -68,8 +80,17 @@ class XmlLogitsProcessor(LogitsProcessor): invalid_tokens_mask = ~valid_tokens_mask scores[batch_idx, invalid_tokens_mask] = float('-inf') continue + + # If the validation failed altogether, this is an invalid path + if not append_result: + # Allow only EOS token if validation fails + scores[batch_idx, :] = float('-inf') + scores[batch_idx, self.eos_token_id] = 0 + continue - for token in batch_processor.get_invalid_tokens(): + # Get tokens that would lead to invalid XML and mask them + invalid_tokens = batch_processor.get_invalid_tokens() + for token in invalid_tokens: if token < scores.shape[1]: scores[batch_idx, token] = float('-inf') @@ -82,9 +103,9 @@ class XmlLogitsProcessor(LogitsProcessor): Returns: A new instance of the processor """ - cloned = XmlLogitsProcessor(self.tokenizer, None) - cloned.eos_token_id = self.eos_token_id - cloned.core = self.core.copy() + # Create a new instance using the existing core + cloned_core = self.core.copy() + cloned = XmlLogitsProcessor(self.tokenizer, core=cloned_core) cloned.prompt_length = self.prompt_length cloned.is_first_call = self.is_first_call return cloned \ No newline at end of file diff --git a/lib/xml_schema_validator/src/lib.rs b/lib/xml_schema_validator/src/lib.rs index 1c697ea..047ef7b 100644 --- a/lib/xml_schema_validator/src/lib.rs +++ b/lib/xml_schema_validator/src/lib.rs @@ -18,6 +18,7 @@ use token::*; #[derive(Clone, Debug)] pub struct XmlSchemaValidator { current_tokens: Vec, + schema: Arc, } /// The Result type used throughout this crate @@ -38,6 +39,7 @@ impl XmlSchemaValidator { Ok(Self { current_tokens, + schema: schema_arc, }) } @@ -51,6 +53,7 @@ impl XmlSchemaValidator { /// Check if the validator has reached the end of the XML pub fn eof(&self) -> bool { + println!("EOF: {:?}", self.current_tokens); self.current_tokens.is_empty() || self.current_tokens.iter().all(Token::is_eof) } @@ -61,11 +64,25 @@ impl XmlSchemaValidator { new_tokens.append(&mut token.append(c)); } if new_tokens.is_empty() { - return Err(Error::InvalidXml("No continuations".to_string())); + return Err(Error::InvalidXml(format!("No valid continuations for character: '{}'", c))); } self.current_tokens = new_tokens; Ok(()) } + + /// Try to validate a continuation character + pub fn can_continue_with(&self, c: char) -> bool { + // Clone the current state and try to append the character + let mut validator_clone = self.clone(); + validator_clone.append_char(c).is_ok() + } + + /// Get a list of all element names in the schema + pub fn get_element_names(&self) -> Vec { + self.schema.xs_element.iter() + .map(|element| element.name.clone()) + .collect() + } } #[cfg(test)] @@ -111,4 +128,472 @@ mod tests { let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); validator.append(input).expect_err("Should fail with mismatched closing tag"); } + + #[test] + fn test_delete_with_attribute() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = r#""#; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_delete_missing_required_attribute() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + // Try to validate an incomplete delete tag that's missing the required id attribute + let input = ""; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + let result = validator.append(input); + assert!(result.is_err(), "Should fail without required id attribute"); + // Check the error message explicitly + if let Err(err) = result { + println!("Error message: {}", err); + assert!(err.to_string().contains("required"), + "Error should mention missing required attribute"); + } + } + + #[test] + fn test_stop_self_closing() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = ""; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_read_stdin_self_closing() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = ""; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_single_with_attributes() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = r#"ls -la"#; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_reasoning_with_content() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = "I should explore the file system for interesting files."; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_write_stdout_with_content() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let input = "Hello world!"; + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append(input).unwrap(); + assert!(validator.eof()); + } + + #[test] + fn test_create_validator() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let validator = XmlSchemaValidator::new(&schema_text); + assert!(validator.is_ok(), "Failed to create validator: {:?}", validator.err()); + } + + #[test] + fn test_get_element_names() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + let names = validator.get_element_names(); + assert!(names.contains(&"reasoning".to_string()), + "Element names should contain 'reasoning', got: {:?}", names); + } + + // Test basic XML fragment validation - debugging the "" failure + #[test] + fn test_append_reasoning_tag() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + // Test character by character for precise failure point + assert!(validator.can_continue_with('<'), "Should accept '<'"); + // Clone at each step to debug exact point of failure + let mut v1 = validator.clone(); + assert!(v1.append_char('<').is_ok(), "Failed to append '<'"); + let mut v2 = v1.clone(); + assert!(v2.append_char('r').is_ok(), "Failed to append 'r'"); + // Continue with each character in "reasoning>" + let mut v3 = v2.clone(); + for c in "easoning>".chars() { + assert!(v3.append_char(c).is_ok(), "Failed to append '{}'", c); + } + // Try the whole string at once + let mut v4 = validator.clone(); + let result = v4.append(""); + assert!(result.is_ok(), "Failed to append '': {:?}", result.err()); + } + + // Test validation of multiple fragments + #[test] + fn test_multiple_append() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + // Test sequential appends + assert!(validator.append("<").is_ok(), "Failed to append '<'"); + assert!(validator.append("reasoning").is_ok(), "Failed to append 'reasoning'"); + assert!(validator.append(">").is_ok(), "Failed to append '>'"); + assert!(validator.append("test content").is_ok(), "Failed to append content"); + assert!(validator.append("").is_ok(), "Failed to append closing tag"); + // Should be at EOF now + assert!(validator.eof(), "Should be at EOF after complete XML"); + } + + // Test token continuation - critical for the token masking issue + #[test] + fn test_can_continue_with() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap(); + // Test at start + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + assert!(validator.can_continue_with('<'), "Should accept '<' at start"); + assert!(!validator.can_continue_with('a'), "Should reject 'a' at start"); + // Test after opening tag + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + validator.append("").unwrap(); + // Any character should be valid inside a mixed content element + assert!(validator.can_continue_with('a'), "Should accept 'a' inside element"); + assert!(validator.can_continue_with('1'), "Should accept '1' inside element"); + assert!(validator.can_continue_with('<'), "Should accept '<' inside element"); + } +} + +#[cfg(test)] +mod diagnostic_tests { + use super::*; + use std::sync::Arc; + + // Simple schema with just the reasoning tag for cleaner debugging + fn get_simple_schema() -> String { + r#" + + + + + + + + + "#.to_string() + } + + // Test every character in an opening tag one by one + #[test] + fn test_char_by_char_open_tag() { + let schema_text = get_simple_schema(); + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + + // Test each character individually + assert!(validator.can_continue_with('<'), "Should accept '<'"); + assert!(validator.append_char('<').is_ok(), "Failed to append '<'"); + + assert!(validator.can_continue_with('r'), "Should accept 'r'"); + assert!(validator.append_char('r').is_ok(), "Failed to append 'r'"); + + for c in "easoning".chars() { + assert!(validator.can_continue_with(c), "Should accept '{}'", c); + assert!(validator.append_char(c).is_ok(), "Failed to append '{}'", c); + } + + // This is where many tests are failing - let's check the state before and after + println!("State before '>': {:?}", validator.current_tokens); + let result = validator.append_char('>'); + println!("Result of append '>': {:?}", result); + if result.is_ok() { + println!("State after '>': {:?}", validator.current_tokens); + } + assert!(result.is_ok(), "Failed to append '>'"); + } + + // Test the token transitions specifically + #[test] + fn test_token_transitions() { + // Create minimal elements for testing token transitions directly + let test_element = Arc::new(schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }); + + // Test the transition from ElementOpenName to ElementOpenEnd + let element_name = token::ElementOpenName::new(Arc::clone(&test_element), "reasoning".to_string()).unwrap(); + let tokens = element_name.append('>'); + assert!(!tokens.is_empty(), "Should produce valid tokens after '>'"); + println!("Token after appending '>' to ElementOpenName: {:?}", tokens[0]); + + // Check what type the token is + match &tokens[0] { + Token::ElementOpen(element_open) => { + println!("Successfully transitioned to ElementOpen: {:?}", element_open); + }, + other => { + panic!("Expected ElementOpen, got {:?}", other); + } + } + } + + // Test element self-closing functionality + #[test] + fn test_element_self_close_transition() { + let test_element = Arc::new(schema::XsElement { + name: "stop".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: None, + text: None, + xs_attribute: None, + xs_sequence: None, + }, + }); + + // Test transition from ElementName to ElementSelfClose + let element_name = token::ElementOpenName::new(Arc::clone(&test_element), "stop".to_string()).unwrap(); + let tokens = element_name.append('/'); + assert!(!tokens.is_empty(), "Should handle '/' after element name"); + + match &tokens[0] { + Token::ElementSelfClose(element_self_close) => { + println!("Successfully transitioned to ElementSelfClose"); + // Now test transition to EndOfFile + let next_tokens = element_self_close.clone().append('>'); + assert!(!next_tokens.is_empty(), "Should handle '>' after '/'"); + + match &next_tokens[0] { + Token::EndOfFile(_) => println!("Successfully transitioned to EndOfFile"), + other => panic!("Expected EndOfFile, got {:?}", other), + } + }, + other => panic!("Expected ElementSelfClose, got {:?}", other), + } + } + + // Test whitespace handling + #[test] + fn test_whitespace_handling() { + let schema_text = get_simple_schema(); + let mut validator = XmlSchemaValidator::new(&schema_text).unwrap(); + + // Test opening tag with whitespace + for c in " + assert!(validator.can_continue_with('>'), "Should accept '>' after whitespace"); + assert!(validator.append_char('>').is_ok(), "Failed to append '>' after whitespace"); + } + + // Test attribute handling + #[test] + fn test_attribute_handling() { + let schema_text = r#" + + + + + + + "#; + + let mut validator = XmlSchemaValidator::new(schema_text).unwrap(); + + // Test opening tag with attribute + for c in "".chars() { + assert!(validator.can_continue_with(c), "Should accept '{}' to close tag", c); + assert!(validator.append_char(c).is_ok(), "Failed to append '{}' to close tag", c); + } + + // Check if we've reached EOF + assert!(validator.eof(), "Should be at EOF after self-closing tag"); + } + + // Test for the specific error patterns seen in the test output + #[test] + fn test_specific_error_cases() { + let schema_text = std::fs::read_to_string("../../action_schema.xsd").unwrap_or_else(|_| get_simple_schema()); + + // Test case 1: Simple opening tag (noticed in `test_element_open`) + let input1 = ""; + let mut validator1 = XmlSchemaValidator::new(&schema_text).unwrap(); + let result1 = validator1.append(input1); + println!("Result appending '': {:?}", result1); + assert!(result1.is_ok(), "Failed on ''"); + + // Test case 2: Element with whitespace (noticed in `test_element_open_whitespace`) + let input2 = ""; + let mut validator2 = XmlSchemaValidator::new(&schema_text).unwrap(); + let result2 = validator2.append(input2); + println!("Result appending '': {:?}", result2); + assert!(result2.is_ok(), "Failed on ''"); + + // Test case 3: Self-closing tags (noticed in `test_read_stdin_self_closing`) + let input3 = ""; + let mut validator3 = XmlSchemaValidator::new(&schema_text).unwrap(); + let result3 = validator3.append(input3); + println!("Result appending '': {:?}", result3); + assert!(result3.is_ok(), "Failed on ''"); + + // Test case 4: Attributes (noticed in `test_single_with_attributes`) + let input4 = r#""#; + let mut validator4 = XmlSchemaValidator::new(&schema_text).unwrap(); + let result4 = validator4.append(input4); + println!("Result appending attribute: {:?}", result4); + assert!(result4.is_ok(), "Failed on attribute handling"); + } + + // Test the basic token state machine with a minimal element + #[test] + fn test_basic_token_operations() { + // Create a minimal element for testing + let element = Arc::new(schema::XsElement { + name: "test".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: None, + }, + }); + + // Create and test each token type + let start_of_file = token::StartOfFile::new(Arc::clone(&element)); + let tokens = start_of_file.append('<'); + assert!(!tokens.is_empty(), "StartOfFile should accept '<'"); + + let element_start = match &tokens[0] { + Token::ElementStart(es) => es.clone(), + _ => panic!("Expected ElementStart"), + }; + + let tokens = element_start.append('t'); + assert!(!tokens.is_empty(), "ElementStart should accept 't'"); + + let element_name = match &tokens[0] { + Token::ElementName(en) => en.clone(), + _ => panic!("Expected ElementName"), + }; + + // This is often where things break + println!("About to append 'e' to ElementName..."); + let tokens = element_name.append('e'); + assert!(!tokens.is_empty(), "ElementName should accept 'e'"); + + // Build the rest of the name + let mut current_token = tokens[0].clone(); + for c in "st".chars() { + let next_tokens = current_token.append(c); + assert!(!next_tokens.is_empty(), "Token should accept '{}'", c); + current_token = next_tokens[0].clone(); + } + + // Now try to append '>' + println!("About to append '>' to token: {:?}", current_token); + let tokens = current_token.append('>'); + println!("Result of appending '>': {:?}", tokens); + assert!(!tokens.is_empty(), "Token should accept '>'"); + + // Verify transition to ElementOpen + match &tokens[0] { + Token::ElementOpen(_) => println!("Successfully transitioned to ElementOpen"), + other => panic!("Expected ElementOpen, got {:?}", other), + } + } + + // This test will help identify issues with token creation for specific elements + #[test] + fn test_element_specific_issues() { + if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") { + let validator = XmlSchemaValidator::new(&schema_text).unwrap(); + + // Get all element names in the schema + let element_names = validator.get_element_names(); + println!("Elements in schema: {:?}", element_names); + + // Test each element separately + for name in &element_names { + println!("\nTesting element: {}", name); + + let mut element_validator = XmlSchemaValidator::new(&schema_text).unwrap(); + let open_tag = format!("<{}>", name); + + // Test character by character + println!("Testing character by character for <{}>", name); + let mut progress = String::new(); + for c in open_tag.chars() { + progress.push(c); + let result = element_validator.can_continue_with(c); + println!(" Can continue with '{}' after '{}': {}", + c, &progress[..progress.len()-1], result); + + if !result { + // This is helpful to diagnose where the process fails + println!(" ❌ Failed on character '{}' for element '{}'", c, name); + break; + } + + if let Err(e) = element_validator.append_char(c) { + println!(" ❌ Failed to append '{}' for element '{}': {:?}", c, name, e); + break; + } + } + + // Test complete tag at once + let mut complete_validator = XmlSchemaValidator::new(&schema_text).unwrap(); + match complete_validator.append(&open_tag) { + Ok(_) => println!(" ✅ Successfully appended <{}>", name), + Err(e) => println!(" ❌ Failed to append <{}>: {:?}", name, e), + } + } + } else { + println!("Schema file not found, skipping element-specific tests"); + } + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/py_xml_logits_processor_core.rs b/lib/xml_schema_validator/src/py_xml_logits_processor_core.rs index 35055cf..1e46659 100644 --- a/lib/xml_schema_validator/src/py_xml_logits_processor_core.rs +++ b/lib/xml_schema_validator/src/py_xml_logits_processor_core.rs @@ -23,10 +23,13 @@ impl XmlLogitsProcessorCore { }) } + /// Append a fragment of XML to the validator + /// Returns true if the append was successful, false otherwise fn append(&mut self, fragment: &str) -> bool { self.xml_schema_validator.append(fragment).is_ok() } + /// Get a list of tokens that would lead to invalid XML fn get_invalid_tokens(&mut self, py: Python<'_>) -> PyResult> { Ok(self.tokens.iter() .filter_map(|token| { @@ -38,21 +41,412 @@ impl XmlLogitsProcessorCore { }) .collect::>()) } + + /// Get a list of tokens that would lead to valid XML + fn get_valid_tokens(&mut self, py: Python<'_>) -> PyResult> { + Ok(self.tokens.iter() + .filter_map(|token| { + if self.xml_schema_validator.clone().append(&token.1).is_ok() { + Some(token.0.clone_ref(py)) + } else { + None + } + }) + .collect::>()) + } + /// Check if the validator has reached the end of the XML fn eof(&self) -> PyResult { Ok(self.xml_schema_validator.eof()) } + /// Create a copy of the processor fn copy(&self, py: Python<'_>) -> PyResult { Ok(Self { xml_schema_validator: self.xml_schema_validator.clone(), tokens: self.tokens.iter().map(|(a, b)|(a.clone_ref(py), b.clone())).collect() }) } + + /// Get a list of all element names in the schema + fn get_element_names(&self) -> PyResult> { + Ok(self.xml_schema_validator.get_element_names()) + } + + /// Check if a specific character is valid as the next character + fn can_continue_with(&self, c: &str) -> PyResult { + if let Some(first_char) = c.chars().next() { + Ok(self.xml_schema_validator.can_continue_with(first_char)) + } else { + Ok(false) + } + } + + /// Reset the validator to its initial state + fn reset(&mut self) -> PyResult<()> { + // Create a new validator with the same schema elements + let schema_text = r#" + + + + + + + + +"#; + + match crate::XmlSchemaValidator::new(schema_text) { + Ok(validator) => { + self.xml_schema_validator = validator; + Ok(()) + }, + Err(e) => Err(pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to reset validator: {}", e + ))), + } + } } /// Register the class with the Python module pub fn register_module(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use pyo3::Python; + use pyo3::types::PyDict; + + fn create_test_processor<'py>(py: Python<'py>) -> PyResult { + let tokens = PyDict::new(py); + // Add some basic tokens + tokens.set_item(0, "<")?; + tokens.set_item(1, "reasoning")?; + tokens.set_item(2, ">")?; + tokens.set_item(3, " ")?; + tokens.set_item(4, "a")?; + tokens.set_item(5, "")?; + + // Simple schema with only reasoning element + let schema_text = r#" + + + + + + + + + "#; + + XmlLogitsProcessorCore::new(tokens.into(), schema_text) + } + + #[test] + fn test_append_valid_xml() { + Python::with_gil(|py| { + let mut processor = create_test_processor(py).unwrap(); + + // Test various valid XML fragments + assert!(processor.append(""), "Should accept opening tag"); + + let mut processor = create_test_processor(py).unwrap(); + assert!(processor.append("test"), + "Should accept complete element with content"); + + // Test with real schema from disk if available + if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") { + let tokens = PyDict::new(py); + tokens.set_item(0, "<").unwrap(); + tokens.set_item(1, "reasoning").unwrap(); + tokens.set_item(2, ">").unwrap(); + + let mut processor = XmlLogitsProcessorCore::new(tokens.into(), &schema_text).unwrap(); + assert!(processor.append(""), + "Should accept with actual schema"); + } + }); + } + + #[test] + fn test_append_invalid_xml() { + Python::with_gil(|py| { + let mut processor = create_test_processor(py).unwrap(); + + // Test invalid XML fragments + assert!(!processor.append(""), "Should reject invalid element"); + + let mut processor = create_test_processor(py).unwrap(); + assert!(!processor.append(""), + "Should reject mismatched tags"); + }); + } + + #[test] + fn test_get_valid_tokens() { + Python::with_gil(|py| { + let mut processor = create_test_processor(py).unwrap(); + + // At start, only "<" is valid + let valid_tokens = processor.get_valid_tokens(py).unwrap(); + + let contains_0 = valid_tokens.iter().any(|obj| { + let obj_id: Option = obj.extract(py).ok(); + obj_id == Some(0) + }); + + assert!(contains_0, "Token '<' should be valid at start"); + assert_eq!(valid_tokens.len(), 1, "Only one token should be valid at start"); + + // After , any character should be valid in mixed content + let mut processor = create_test_processor(py).unwrap(); + assert!(processor.append(""), "Should be able to append "); + + let valid_tokens = processor.get_valid_tokens(py).unwrap(); + + let contains_3 = valid_tokens.iter().any(|obj| { + let obj_id: Option = obj.extract(py).ok(); + obj_id == Some(3) + }); + + let contains_4 = valid_tokens.iter().any(|obj| { + let obj_id: Option = obj.extract(py).ok(); + obj_id == Some(4) + }); + + assert!(contains_3, "Token ' ' should be valid inside mixed content"); + assert!(contains_4, "Token 'a' should be valid inside mixed content"); + }); + } + + #[test] + fn test_eof_detection() { + Python::with_gil(|py| { + let processor = create_test_processor(py).unwrap(); + + // At start, not at EOF + assert!(!processor.eof().unwrap(), "Should not be at EOF at start"); + + // After complete document, should be at EOF + let mut processor = create_test_processor(py).unwrap(); + // Use a properly formed XML document that matches the schema + assert!(processor.append("test"), + "Should accept complete document"); + + // DEBUG: Print the token state to see why EOF detection fails + println!("Current tokens: {:?}", processor.xml_schema_validator.current_tokens); + + assert!(processor.eof().unwrap(), "Should be at EOF after complete document"); + + // After opening tag, should not be at EOF + let mut processor = create_test_processor(py).unwrap(); + assert!(processor.append(""), "Should accept opening tag"); + assert!(!processor.eof().unwrap(), "Should not be at EOF after opening tag"); + }); + } + + #[test] + fn test_can_continue_with() { + Python::with_gil(|py| { + let processor = create_test_processor(py).unwrap(); + + // At start, only '<' is valid + assert!(processor.can_continue_with("<").unwrap(), "Should accept '<' at start"); + assert!(!processor.can_continue_with("a").unwrap(), "Should reject 'a' at start"); + + // Test after opening tag + let mut processor = create_test_processor(py).unwrap(); + assert!(processor.append(""), "Should accept opening tag"); + + // In mixed content, any character should be valid + assert!(processor.can_continue_with("a").unwrap(), + "Should accept 'a' in mixed content"); + assert!(processor.can_continue_with("<").unwrap(), + "Should accept '<' in mixed content"); + }); + } + + #[test] + fn test_copy() { + Python::with_gil(|py| { + let processor = create_test_processor(py).unwrap(); + + // Make a copy + let copy = processor.copy(py).unwrap(); + + // Original and copy should behave the same + assert_eq!(processor.eof().unwrap(), copy.eof().unwrap(), + "Copy should have same EOF state"); + + // Modify copy, original should be unchanged + let mut copy = processor.copy(py).unwrap(); + assert!(copy.append(""), "Copy should accept XML"); + assert!(!processor.eof().unwrap(), "Original should be unchanged"); + assert!(!copy.eof().unwrap(), "Copy should be modified"); + }); + } + + #[test] + fn test_get_element_names() { + Python::with_gil(|py| { + let processor = create_test_processor(py).unwrap(); + + let names = processor.get_element_names().unwrap(); + assert!(names.contains(&"reasoning".to_string()), + "Should contain 'reasoning' element"); + assert_eq!(names.len(), 1, "Should have exactly one element"); + + // Test with real schema from disk if available + if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") { + let tokens = PyDict::new(py); + tokens.set_item(0, "<").unwrap(); + + let processor = XmlLogitsProcessorCore::new(tokens.into(), &schema_text).unwrap(); + let names = processor.get_element_names().unwrap(); + + // Print out all elements from schema for debugging + println!("Elements in schema: {:?}", names); + + // Check if reasoning exists + assert!(names.contains(&"reasoning".to_string()), + "Real schema should contain 'reasoning'"); + } + }); + } + + #[test] + fn test_debug_reasoning_tag_validation() { + Python::with_gil(|py| { + // Create a simple schema with just the reasoning element for testing + let schema_text = r#" + + + + + + + + + "#; + + // Create a minimal token dictionary + let tokens = PyDict::new(py); + tokens.set_item(0, "<").unwrap(); + tokens.set_item(1, "r").unwrap(); + tokens.set_item(2, "e").unwrap(); + tokens.set_item(3, "a").unwrap(); + tokens.set_item(4, "s").unwrap(); + tokens.set_item(5, "o").unwrap(); + tokens.set_item(6, "n").unwrap(); + tokens.set_item(7, "i").unwrap(); + tokens.set_item(8, "n").unwrap(); + tokens.set_item(9, "g").unwrap(); + tokens.set_item(10, ">").unwrap(); + + // Create processor with this schema and tokens + let processor = XmlLogitsProcessorCore::new(tokens.clone().into(), schema_text).unwrap(); + + // Test basic operations to verify core functionality works + let element_names = processor.get_element_names().unwrap(); + println!("Element names in test schema: {:?}", element_names); + assert!(element_names.contains(&"reasoning".to_string()), + "Test schema should contain 'reasoning'"); + + // Now try to append each character of "" one by one + let chars = vec!['<', 'r', 'e', 'a', 's', 'o', 'n', 'i', 'n', 'g', '>']; + let mut full_text = String::new(); + let processor = processor.copy(py).unwrap(); + + for c in chars { + full_text.push(c); + let fragment = c.to_string(); + let result = processor.can_continue_with(&fragment).unwrap(); + println!("Can continue with '{}' after '{}': {}", + c, &full_text[..full_text.len()-1], result); + + // Create a new processor to test appending the fragment built so far + let mut fresh_processor = XmlLogitsProcessorCore::new(tokens.clone().into(), schema_text).unwrap(); + let append_result = fresh_processor.append(&full_text); + println!("Appending full text '{}': {}", full_text, append_result); + } + + // Now test with actual schema from disk if available + if let Ok(actual_schema) = std::fs::read_to_string("../../action_schema.xsd") { + println!("\nTesting with actual schema from disk:"); + + let processor = XmlLogitsProcessorCore::new(tokens.clone().into(), &actual_schema).unwrap(); + let element_names = processor.get_element_names().unwrap(); + println!("Element names in actual schema: {:?}", element_names); + + if element_names.contains(&"reasoning".to_string()) { + println!("'reasoning' element found in actual schema"); + + // Test appending "" to a fresh processor + let mut processor = processor.copy(py).unwrap(); + let result = processor.append(""); + println!("Appending '' to fresh processor: {}", result); + + // Test character by character + let mut text = String::new(); + for c in "".chars() { + text.push(c); + let result = XmlLogitsProcessorCore::new(tokens.clone().into(), &actual_schema).unwrap() + .append(&text); + println!("Appending '{}' to fresh processor: {}", text, result); + } + } else { + println!("Warning: 'reasoning' element NOT found in actual schema!"); + } + } + }); + } + + #[test] + fn test_debug_token_processing() { + Python::with_gil(|py| { + if let Ok(schema_text) = std::fs::read_to_string("../../action_schema.xsd") { + // Create a dictionary mapping token IDs to their string values + // Simulating how the tokenizer vocabulary works + let tokens = PyDict::new(py); + tokens.set_item(1000, "<").unwrap(); + tokens.set_item(1001, "reasoning").unwrap(); + tokens.set_item(1002, ">").unwrap(); + tokens.set_item(1003, " ").unwrap(); + tokens.set_item(1004, "test").unwrap(); + tokens.set_item(1005, "").unwrap(); + + let mut processor = XmlLogitsProcessorCore::new(tokens.clone().into(), &schema_text).unwrap(); + + // Test which tokens are valid at the start + let invalid_tokens = processor.get_invalid_tokens(py).unwrap(); + let valid_tokens = processor.get_valid_tokens(py).unwrap(); + + println!("At start - valid tokens count: {}", valid_tokens.len()); + println!("At start - invalid tokens count: {}", invalid_tokens.len()); + + // Create a new processor and progress to after "" + let mut processor = XmlLogitsProcessorCore::new(tokens.into(), &schema_text).unwrap(); + let result = processor.append(""); + println!("Append '': {}", result); + + if result { + // Now get valid/invalid tokens after "" + let invalid_tokens = processor.get_invalid_tokens(py).unwrap(); + let valid_tokens = processor.get_valid_tokens(py).unwrap(); + + println!("After '' - valid tokens count: {}", valid_tokens.len()); + println!("After '' - invalid tokens count: {}", invalid_tokens.len()); + + // At this point we'd expect to have more valid tokens than at the start + // since mixed content allows any text + } + } else { + println!("Warning: Couldn't find schema file for testing"); + } + }); + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/py_xml_schema_validator.rs b/lib/xml_schema_validator/src/py_xml_schema_validator.rs index 0849e42..1cc8c31 100644 --- a/lib/xml_schema_validator/src/py_xml_schema_validator.rs +++ b/lib/xml_schema_validator/src/py_xml_schema_validator.rs @@ -17,6 +17,10 @@ impl XmlSchemaValidator { Ok(Self { validator }) } + /// Append a fragment of XML to the validator + /// Returns a tuple containing: + /// - A boolean indicating success + /// - An optional error message fn append(&mut self, fragment: &str) -> PyResult<(bool, Option)> { match self.validator.append(fragment) { Ok(()) => PyResult::Ok((true, None)), @@ -24,15 +28,40 @@ impl XmlSchemaValidator { } } + /// Check if the validator has reached the end of the XML fn eof(&self) -> PyResult { Ok(self.validator.eof()) } + /// Create a deep copy of this validator fn copy(&self) -> PyResult { Ok(Self { validator: self.validator.clone(), }) } + + /// Get a list of all element names in the schema + fn get_element_names(&self) -> PyResult> { + Ok(self.validator.get_element_names()) + } + + /// Check if a specific character is valid as the next character + fn can_continue_with(&self, c: &str) -> PyResult { + if let Some(first_char) = c.chars().next() { + Ok(self.validator.can_continue_with(first_char)) + } else { + Ok(false) + } + } + + /// Validate an entire XML string and check if it's valid according to the schema + fn validate(&self, xml_string: &str) -> PyResult { + let mut validator_clone = self.validator.clone(); + match validator_clone.append(xml_string) { + Ok(()) => Ok(validator_clone.eof()), // Valid only if we reached EOF + Err(_) => Ok(false), // Invalid XML + } + } } /// Register the class with the Python module diff --git a/lib/xml_schema_validator/src/token/attribute_name.rs b/lib/xml_schema_validator/src/token/attribute_name.rs new file mode 100644 index 0000000..6dbe159 --- /dev/null +++ b/lib/xml_schema_validator/src/token/attribute_name.rs @@ -0,0 +1,150 @@ +use std::sync::Arc; +use crate::*; + +/// Represents an attribute name of an XML element +#[derive(Clone, Debug)] +pub struct AttributeName { + /// The element that owns this attribute + element: Arc, + /// Buffer containing the attribute name characters processed so far + buffer: String, +} + +impl AttributeName { + /// Create a new AttributeName with an empty buffer + pub fn new(element: Arc) -> Self { + Self { + element, + buffer: String::new(), + } + } + + /// Create a new AttributeName with the given buffer + pub fn with_buffer(element: Arc, buffer: String) -> Self { + Self { + element, + buffer, + } + } + + /// Append a character to the attribute name and return possible continuations + pub fn append(self, c: char) -> Vec { + if c.is_whitespace() && self.buffer.is_empty() { + return vec![Token::AttributeName(self)]; + } else if c == '=' { + // Found the equals sign, transition to attribute value + // We need to verify that this attribute name is valid for this element + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + for attr in attributes { + if attr.name == self.buffer { + // Valid attribute, transition to attribute value + return vec![Token::AttributeValue(AttributeValue::new( + Arc::clone(&self.element), + self.buffer, + ))]; + } + } + } + // Invalid attribute name + vec![] + } else { + // Continue building the attribute name + let new_buffer = self.buffer + &c.to_string(); + + // Check if this could be a valid attribute for this element + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + for attr in attributes { + if attr.name.starts_with(&new_buffer) { + // This could be a valid attribute, continue parsing + return vec![Token::AttributeName(AttributeName::with_buffer( + Arc::clone(&self.element), + new_buffer, + ))]; + } + } + } + + // No matching attribute found + vec![] + } + } +} + +#[cfg(test)] +mod attribute_name_tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_element_with_attributes() -> Arc { + let element = schema::XsElement { + name: "delete".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: None, + text: None, + xs_attribute: Some(vec![ + schema::XsAttribute { + name: "id".to_string(), + xs_attribute_type: "xs:string".to_string(), + xs_attribute_use: "required".to_string(), + } + ]), + xs_sequence: None, + }, + }; + Arc::new(element) + } + + #[test] + fn test_attribute_name_append() { + let element = create_element_with_attributes(); + let token = AttributeName::new(Arc::clone(&element)); + + // Build attribute name character by character + let mut current_token = token; + for c in "id".chars() { + let next_tokens = current_token.append(c); + assert!(!next_tokens.is_empty(), "Should accept attribute name character '{}'", c); + + match &next_tokens[0] { + Token::AttributeName(next) => current_token = next.clone(), + _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]), + } + } + + // Test equals sign + let next_tokens = current_token.append('='); + assert!(!next_tokens.is_empty(), "Should accept '=' after attribute name"); + + match &next_tokens[0] { + Token::AttributeValue(_) => (), + _ => panic!("Expected AttributeValue, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_attribute_name_whitespace() { + let element = create_element_with_attributes(); + let token = AttributeName::new(Arc::clone(&element)); + + // Test with whitespace + let next_tokens = token.append(' '); + assert!(!next_tokens.is_empty(), "Should accept whitespace in attribute name"); + + match &next_tokens[0] { + Token::AttributeName(_) => (), + _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_invalid_attribute_name() { + let element = create_element_with_attributes(); + let token = AttributeName::new(Arc::clone(&element)); + + // Test with invalid attribute name + let next_tokens = token.append('x'); // 'x' doesn't start any valid attribute + assert!(next_tokens.is_empty(), "Should reject invalid attribute name"); + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/attribute_value.rs b/lib/xml_schema_validator/src/token/attribute_value.rs new file mode 100644 index 0000000..27698fe --- /dev/null +++ b/lib/xml_schema_validator/src/token/attribute_value.rs @@ -0,0 +1,215 @@ +use std::sync::Arc; +use crate::*; + +/// Represents an attribute value of an XML element +#[derive(Clone, Debug)] +pub struct AttributeValue { + /// The element that owns this attribute + element: Arc, + /// The name of the attribute being processed + attribute_name: String, + /// Buffer containing the attribute value characters processed so far + buffer: String, + /// Whether we've encountered the opening quote + in_quotes: bool, + /// Whether we've seen the closing quote + closed: bool, +} + +impl AttributeValue { + /// Create a new AttributeValue + pub fn new(element: Arc, attribute_name: String) -> Self { + Self { + element, + attribute_name, + buffer: String::new(), + in_quotes: false, + closed: false, + } + } + + /// Append a character to the attribute value and return possible continuations + pub fn append(self, c: char) -> Vec { + if self.closed { + // We've already closed the attribute value with a quote + if c.is_whitespace() { + // Check if there are more attributes to parse + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + // If there are other required attributes that haven't been set yet + for attr in attributes { + if attr.name != self.attribute_name && attr.xs_attribute_use == "required" { + return vec![Token::AttributeName(AttributeName::new(Arc::clone(&self.element)))]; + } + } + } + + // No more required attributes, whitespace after attribute value, continue to next state + return vec![ + Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element))), + Token::ElementSelfClose(ElementSelfClose::new()) + ]; + } else if c == '>' { + // End of opening tag + return vec![Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element)))]; + } else if c == '/' { + // Beginning of self-closing tag + return vec![Token::ElementSelfClose(ElementSelfClose::new())]; + } else { + // Unexpected character after attribute value + return vec![]; + } + } + + if !self.in_quotes { + if c.is_whitespace() { + // Skip whitespace before the quotes + return vec![Token::AttributeValue(Self { + element: Arc::clone(&self.element), + attribute_name: self.attribute_name, + buffer: self.buffer, + in_quotes: false, + closed: false, + })]; + } else if c == '"' || c == '\'' { + // Start of quotes + return vec![Token::AttributeValue(Self { + element: Arc::clone(&self.element), + attribute_name: self.attribute_name, + buffer: self.buffer, + in_quotes: true, + closed: false, + })]; + } else { + // Unexpected character before quotes + return vec![]; + } + } else { + // We're inside quotes + if c == '"' || c == '\'' { + // End of quotes, validate the attribute value + // In a real implementation, we'd check if the value is valid for this attribute type + return vec![Token::AttributeValue(Self { + element: Arc::clone(&self.element), + attribute_name: self.attribute_name, + buffer: self.buffer, + in_quotes: true, + closed: true, + })]; + } else { + // Continue building the attribute value + let new_buffer = self.buffer + &c.to_string(); + return vec![Token::AttributeValue(Self { + element: Arc::clone(&self.element), + attribute_name: self.attribute_name, + buffer: new_buffer, + in_quotes: true, + closed: false, + })]; + } + } + } +} + +#[cfg(test)] +mod attribute_value_tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_element_with_attributes() -> Arc { + let element = schema::XsElement { + name: "delete".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: None, + text: None, + xs_attribute: Some(vec![ + schema::XsAttribute { + name: "id".to_string(), + xs_attribute_type: "xs:string".to_string(), + xs_attribute_use: "required".to_string(), + } + ]), + xs_sequence: None, + }, + }; + Arc::new(element) + } + + #[test] + fn test_attribute_value_quotes() { + let element = create_element_with_attributes(); + let token = AttributeValue::new(Arc::clone(&element), "id".to_string()); + + // Test with opening quote + let next_tokens = token.append('"'); + assert!(!next_tokens.is_empty(), "Should accept opening quote"); + + let token = next_tokens[0].clone(); + match token.clone() { + Token::AttributeValue(value) => { + assert!(value.in_quotes, "Should mark as in quotes"); + assert!(!value.closed, "Should not be closed yet"); + }, + _ => panic!("Expected AttributeValue, got {:?}", token), + } + + // Test with content inside quotes + let next_tokens = token.append('a'); + assert!(!next_tokens.is_empty(), "Should accept content inside quotes"); + + let token = next_tokens[0].clone(); + + // Test with closing quote + let next_tokens = token.append('"'); + assert!(!next_tokens.is_empty(), "Should accept closing quote"); + + let token = next_tokens[0].clone(); + match token { + Token::AttributeValue(value) => { + assert!(value.closed, "Should be closed"); + }, + _ => panic!("Expected AttributeValue, got {:?}", token), + } + } + + #[test] + fn test_after_attribute_value_closed() { + let element = create_element_with_attributes(); + let mut token = AttributeValue::new(Arc::clone(&element), "id".to_string()); + token.in_quotes = true; + token.closed = true; + + // Test with space after closed attribute value + let next_tokens = token.append(' '); + assert!(!next_tokens.is_empty(), "Should accept space after closed attribute"); + + // Test with > after closed attribute value + let token = AttributeValue::new(Arc::clone(&element), "id".to_string()); + let mut token = token; + token.in_quotes = true; + token.closed = true; + + let next_tokens = token.append('>'); + assert!(!next_tokens.is_empty(), "Should accept '>' after closed attribute"); + + match &next_tokens[0] { + Token::ElementOpen(_) => (), + _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), + } + + // Test with / after closed attribute value (for self-closing) + let token = AttributeValue::new(Arc::clone(&element), "id".to_string()); + let mut token = token; + token.in_quotes = true; + token.closed = true; + + let next_tokens = token.append('/'); + assert!(!next_tokens.is_empty(), "Should accept '/' after closed attribute"); + + match &next_tokens[0] { + Token::ElementSelfClose(_) => (), + _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]), + } + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_close_end.rs b/lib/xml_schema_validator/src/token/element_close_end.rs deleted file mode 100644 index 1fc96da..0000000 --- a/lib/xml_schema_validator/src/token/element_close_end.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::*; - -/// Represents the end of a closing element tag -#[derive(Clone, Debug)] -pub struct ElementCloseEnd { -} - -impl ElementCloseEnd { - pub fn new() -> Self { - Self { - } - } - - pub fn append(self, c: char) -> Vec { - if '>' == c { - vec![Token::EndOfFile(EndOfFile::new())] - } else { - vec![] - } - } -} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_close_name.rs b/lib/xml_schema_validator/src/token/element_close_name.rs index 391f627..67b652a 100644 --- a/lib/xml_schema_validator/src/token/element_close_name.rs +++ b/lib/xml_schema_validator/src/token/element_close_name.rs @@ -21,32 +21,117 @@ impl ElementCloseName { } pub fn append(self, c: char) -> Vec { + if self.element.name == self.buffer && c.is_whitespace() { + return vec![Token::ElementCloseName(self)]; + } + if self.element.name == self.buffer && c == '>' { + return vec![Token::EndOfFile(EndOfFile::new())]; + } let new_buffer = self.buffer + &c.to_string(); - - // If still building the element name if self.element.name.starts_with(&new_buffer) { - // If we've matched the full name - if self.element.name == new_buffer { - // Now we're expecting '>' to close the tag - if c.is_whitespace() { - // Handle whitespace after the element name - vec![Token::Whitespace(Whitespace::new( - vec![Token::ElementCloseEnd(ElementCloseEnd::new())] - ))] - } else { - // Directly transition to the closed state - vec![Token::ElementCloseEnd(ElementCloseEnd::new())] - } - } else { - // Still building the element name - vec![Token::ElementCloseName(ElementCloseName::new( - Arc::clone(&self.element), - new_buffer, - ))] - } + return vec![Token::ElementCloseName(ElementCloseName::new( + Arc::clone(&self.element), + new_buffer, + ))]; } else { - // The character doesn't match what we expect for this element name vec![] } } +} + +#[cfg(test)] +mod element_close_name_tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_close_name_partial() { + let element = create_test_element(); + let token = ElementCloseName::new(Arc::clone(&element), "r".to_string()); + + // Test with next character in name + let next_tokens = token.append('e'); + assert!(!next_tokens.is_empty(), "Should accept next character in name"); + + match &next_tokens[0] { + Token::ElementCloseName(name) => { + assert_eq!(name.buffer, "re", "Buffer should contain 're'"); + }, + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_name_complete() { + let element = create_test_element(); + let token = ElementCloseName::new(Arc::clone(&element), "reasonin".to_string()); + + // Test with last character of name + let next_tokens = token.append('g'); + assert!(!next_tokens.is_empty(), "Should accept last character of name"); + + match &next_tokens[0] { + Token::ElementCloseName(name) => { + assert_eq!(name.buffer, "reasoning", "Buffer should contain 'reasoning'"); + }, + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + + // Test with > after complete name + let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string()); + let next_tokens = token.append('>'); + assert!(!next_tokens.is_empty(), "Should accept '>' after complete name"); + + match &next_tokens[0] { + Token::EndOfFile(_) => (), + _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_name_whitespace() { + let element = create_test_element(); + let token = ElementCloseName::new(Arc::clone(&element), "reasoning".to_string()); + + // Test with whitespace after complete name + let next_tokens = token.append(' '); + assert!(!next_tokens.is_empty(), "Should accept whitespace after complete name"); + + match &next_tokens[0] { + Token::ElementCloseName(_) => (), + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_name_invalid() { + let element = create_test_element(); + let token = ElementCloseName::new(Arc::clone(&element), "r".to_string()); + + // Test with invalid next character + let next_tokens = token.append('x'); + assert!(next_tokens.is_empty(), "Should reject invalid character"); + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_close_start.rs b/lib/xml_schema_validator/src/token/element_close_start.rs index 7e52c80..ac77966 100644 --- a/lib/xml_schema_validator/src/token/element_close_start.rs +++ b/lib/xml_schema_validator/src/token/element_close_start.rs @@ -34,4 +34,76 @@ impl ElementCloseStart { } } } +} + +#[cfg(test)] +mod element_close_start_tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_close_start_slash() { + let element = create_test_element(); + let token = ElementCloseStart::new(Arc::clone(&element)); + + // Test with slash + let next_tokens = token.append('/'); + assert!(!next_tokens.is_empty(), "Should accept '/'"); + + match &next_tokens[0] { + Token::ElementCloseStart(start) => { + assert!(start.has_slash, "Should mark as having slash"); + }, + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_start_element_name() { + let element = create_test_element(); + let mut token = ElementCloseStart::new(Arc::clone(&element)); + token.has_slash = true; + + // Test with first character of element name + let next_tokens = token.append('r'); + assert!(!next_tokens.is_empty(), "Should accept first letter of element name"); + + match &next_tokens[0] { + Token::ElementCloseName(_) => (), + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_close_start_invalid_name() { + let element = create_test_element(); + let mut token = ElementCloseStart::new(Arc::clone(&element)); + token.has_slash = true; + + // Test with invalid first character of element name + let next_tokens = token.append('x'); + assert!(next_tokens.is_empty(), "Should reject invalid first letter"); + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_open_end.rs b/lib/xml_schema_validator/src/token/element_open_end.rs index 467b0bb..5db25c1 100644 --- a/lib/xml_schema_validator/src/token/element_open_end.rs +++ b/lib/xml_schema_validator/src/token/element_open_end.rs @@ -15,7 +15,7 @@ impl ElementOpenEnd { } pub fn append(self, c: char) -> Vec { - // Check if the element allows text content (mixed="true") + // Check if the element is a mixed content element that allows text let allows_text = self.element.xs_complex_type.mixed .as_ref() .map(|mixed| mixed == "true") @@ -23,14 +23,140 @@ impl ElementOpenEnd { if allows_text { if c.is_whitespace() { - vec![Token::Whitespace(Whitespace::new( - vec![Token::TextContent(TextContent::new(Arc::clone(&self.element)))] - ))] + // Handle whitespace in mixed content + vec![ + Token::TextContent(TextContent::new(Arc::clone(&self.element))), + Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element))) + ] + } else if c == '<' { + // Start of a closing tag or nested element + vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))] } else { + // Text content vec![Token::TextContent(TextContent::new(Arc::clone(&self.element)))] } } else { - vec![] + // Element doesn't allow text content, only expect closing tag + if c == '<' { + vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))] + } else if c.is_whitespace() { + vec![Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&self.element)))] + } else { + // Unexpected content in non-mixed element + vec![] + } } } +} + +#[cfg(test)] +mod element_open_end_tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + fn create_element_non_mixed() -> Arc { + let element = schema::XsElement { + name: "element".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: None, // Not mixed content + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_append_text_in_mixed_content() { + let element = create_test_element(); + let token = ElementOpenEnd::new(Arc::clone(&element)); + + // Test appending text in mixed content element + let next_tokens = token.append('a'); + assert!(!next_tokens.is_empty(), "Should allow text content in mixed element"); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_closing_tag_start() { + let element = create_test_element(); + let token = ElementOpenEnd::new(Arc::clone(&element)); + + // Test starting a closing tag + let next_tokens = token.append('<'); + assert!(!next_tokens.is_empty(), "Should allow closing tag start"); + + match &next_tokens[0] { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_whitespace_in_mixed() { + let element = create_test_element(); + let token = ElementOpenEnd::new(Arc::clone(&element)); + + // Test whitespace in mixed content + let next_tokens = token.append(' '); + assert!(!next_tokens.is_empty(), "Should allow whitespace in mixed content"); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_in_non_mixed() { + let element = create_element_non_mixed(); + let token = ElementOpenEnd::new(Arc::clone(&element)); + + // Test appending text in non-mixed content (should reject) + let next_tokens = token.clone().append('a'); + assert!(next_tokens.is_empty(), "Should reject text in non-mixed element"); + + // Test whitespace (should accept leading to closing tag) + let next_tokens = token.clone().append(' '); + assert!(!next_tokens.is_empty(), "Should allow whitespace in non-mixed"); + + // Test closing tag start + let next_tokens = token.append('<'); + assert!(!next_tokens.is_empty(), "Should allow closing tag start in non-mixed"); + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_open_name.rs b/lib/xml_schema_validator/src/token/element_open_name.rs index bd221a8..74a9a76 100644 --- a/lib/xml_schema_validator/src/token/element_open_name.rs +++ b/lib/xml_schema_validator/src/token/element_open_name.rs @@ -16,25 +16,218 @@ impl ElementOpenName { element, }) } else { - Err(Error::NotImplemented) + Err(Error::InvalidXml(format!("Element name '{}' doesn't match '{}'", buffer, element.name))) } } pub fn append(self, c: char) -> Vec { - let buffer = self.buffer + &c.to_string(); - if self.element.name.starts_with(buffer.as_str()) { - vec![Token::ElementName(ElementOpenName { - buffer, - element: Arc::clone(&self.element), - })] - } else if c.is_whitespace() { - vec![Token::Whitespace(Whitespace::new( - vec![Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element)))] - ))] - } else if '>' == c { - vec![Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element)))] + if c.is_whitespace() { + // Check if we've matched the full element name + if self.element.name == self.buffer { + // Element name is complete, handle whitespace after name + + // Check if the element has attributes + if let Some(attributes) = &self.element.xs_complex_type.xs_attribute { + if !attributes.is_empty() { + // Transition to attribute name parsing + return vec![Token::AttributeName(AttributeName::new(Arc::clone(&self.element)))]; + } + } + + // No attributes, continue to element open or self-close + return vec![ + Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element))), + Token::ElementSelfClose(ElementSelfClose::new()) + ]; + } else { + // Whitespace in the middle of an element name is invalid + return vec![]; + } + } else if c == '>' { + // Check if we've matched the full element name + if self.element.name == self.buffer { + // Element name is complete, end of opening tag + return vec![Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&self.element)))]; + } else { + // '>' before full element name is invalid + return vec![]; + } + } else if c == '/' { + // Check if we've matched the full element name + if self.element.name == self.buffer { + // Element name is complete, self-closing tag + return vec![Token::ElementSelfClose(ElementSelfClose::new())]; + } else { + // '/' before full element name is invalid + return vec![]; + } } else { - vec![] + // Add character to buffer and check if we're still building a valid element name + let new_buffer = self.buffer.clone() + &c.to_string(); + + if self.element.name.starts_with(&new_buffer) { + // Still building the element name + return vec![Token::ElementName(ElementOpenName { + buffer: new_buffer, + element: Arc::clone(&self.element), + })]; + } else { + // Character doesn't match what we expect for this element name + return vec![]; + } } } +} + +#[cfg(test)] +mod element_open_name_tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + fn create_element_with_attributes() -> Arc { + let element = schema::XsElement { + name: "delete".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: None, + text: None, + xs_attribute: Some(vec![ + schema::XsAttribute { + name: "id".to_string(), + xs_attribute_type: "xs:string".to_string(), + xs_attribute_use: "required".to_string(), + } + ]), + xs_sequence: None, + }, + }; + Arc::new(element) + } + + #[test] + fn test_append_character_by_character() { + let element = create_test_element(); + + // Test each character in "reasoning" + let mut buffer = String::new(); + for c in "reasoning".chars() { + buffer.push(c); + let result = ElementOpenName::new(Arc::clone(&element), buffer.clone()); + assert!(result.is_ok(), "Failed to create ElementOpenName with buffer '{}'", buffer); + } + } + + #[test] + fn test_append_full_name() { + let element = create_test_element(); + + // Test with the full element name + let result = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()); + assert!(result.is_ok(), "Failed to create ElementOpenName with full element name"); + + let element_name = result.unwrap(); + + // Test transition after full name with '>' + let next_tokens = element_name.append('>'); + assert!(!next_tokens.is_empty(), "Should accept '>' after full element name"); + + match &next_tokens[0] { + Token::ElementOpen(_) => (), + _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_space_after_name() { + let element = create_test_element(); + + // Test with the full element name + let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap(); + + // Test transition after full name with space + let next_tokens = element_name.append(' '); + assert!(!next_tokens.is_empty(), "Should accept space after full element name"); + + match &next_tokens[0] { + Token::ElementOpen(_) => (), + _ => panic!("Expected ElementOpen, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_with_attributes() { + let element = create_element_with_attributes(); + + // Test with the full element name + let element_name = ElementOpenName::new(Arc::clone(&element), "delete".to_string()).unwrap(); + + // Test transition after full name with space (expecting attribute name) + let next_tokens = element_name.append(' '); + assert!(!next_tokens.is_empty(), "Should accept space after element name with attributes"); + + // The space should lead to a whitespace token, which should then lead to attribute name + match &next_tokens[0] { + Token::AttributeName(_) => (), + _ => panic!("Expected AttributeName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_append_slash_after_name() { + let element = create_test_element(); + + // Test with the full element name + let element_name = ElementOpenName::new(Arc::clone(&element), "reasoning".to_string()).unwrap(); + + // Test transition after full name with / + let next_tokens = element_name.append('/'); + assert!(!next_tokens.is_empty(), "Should accept '/' after full element name"); + + match &next_tokens[0] { + Token::ElementSelfClose(_) => (), + _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_invalid_continuation() { + let element = create_test_element(); + + // Test with a partial element name and an invalid continuation character + let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap(); + let next_tokens = element_name.append('x'); + assert!(next_tokens.is_empty(), "Should reject invalid character in element name"); + + // Test whitespace in middle of name - need to create a new instance since append consumes self + let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap(); + let next_tokens = element_name.append(' '); + assert!(next_tokens.is_empty(), "Should reject whitespace in middle of element name"); + + // Test > in middle of name - need to create a new instance since append consumes self + let element_name = ElementOpenName::new(Arc::clone(&element), "reas".to_string()).unwrap(); + let next_tokens = element_name.append('>'); + assert!(next_tokens.is_empty(), "Should reject '>' in middle of element name"); + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/element_self_close.rs b/lib/xml_schema_validator/src/token/element_self_close.rs new file mode 100644 index 0000000..e40467f --- /dev/null +++ b/lib/xml_schema_validator/src/token/element_self_close.rs @@ -0,0 +1,68 @@ +use crate::*; + +/// Represents a self-closing XML element +#[derive(Clone, Debug)] +pub struct ElementSelfClose { +} + +impl ElementSelfClose { + /// Create a new self-closing element + pub fn new() -> Self { + Self { + } + } + + /// Append a character to the self-closing tag and return possible continuations + pub fn append(self, c: char) -> Vec { + if c == '>' { + vec![Token::EndOfFile(EndOfFile::new())] + } else if c.is_whitespace() { + vec![Token::ElementSelfClose(self)] + } else { + vec![] + } + } +} + +#[cfg(test)] +mod element_self_close_tests { + use super::*; + + #[test] + fn test_self_close_with_greater_than() { + let token = ElementSelfClose::new(); + + // Test closing with > + let next_tokens = token.append('>'); + assert!(!next_tokens.is_empty(), "Should accept '>' to close self-closing tag"); + assert_eq!(next_tokens.len(), 1, "Should have exactly one token after '>'"); + + match &next_tokens[0] { + Token::EndOfFile(_) => (), + _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_self_close_with_whitespace() { + let token = ElementSelfClose::new(); + + // Test with whitespace + let next_tokens = token.append(' '); + assert!(!next_tokens.is_empty(), "Should accept whitespace in self-closing tag"); + + match &next_tokens[0] { + Token::ElementSelfClose(_) => (), + _ => panic!("Expected ElementSelfClose, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_self_close_with_invalid_char() { + let token = ElementSelfClose::new(); + + // Test with invalid character + let next_tokens = token.append('a'); + assert!(next_tokens.is_empty(), "Should reject invalid character in self-closing tag"); + } +} \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/end_of_file.rs b/lib/xml_schema_validator/src/token/end_of_file.rs index 124abda..8e9a071 100644 --- a/lib/xml_schema_validator/src/token/end_of_file.rs +++ b/lib/xml_schema_validator/src/token/end_of_file.rs @@ -13,4 +13,18 @@ impl EndOfFile { pub fn append(self, _c: char) -> Vec { vec![] } +} + +#[cfg(test)] +mod end_of_file_tests { + use super::*; + + #[test] + fn test_end_of_file_any_char() { + let token = EndOfFile::new(); + + // Test with any character (should reject) + let next_tokens = token.append('a'); + assert!(next_tokens.is_empty(), "Should reject any character at EOF"); + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/mod.rs b/lib/xml_schema_validator/src/token/mod.rs index 09188ed..a205cdc 100644 --- a/lib/xml_schema_validator/src/token/mod.rs +++ b/lib/xml_schema_validator/src/token/mod.rs @@ -1,4 +1,3 @@ -mod element_close_end; mod element_close_name; mod element_close_start; mod element_open_name; @@ -7,9 +6,10 @@ mod element_open_start; mod end_of_file; mod start_of_file; mod text_content; -mod whitespace; +mod attribute_name; +mod attribute_value; +mod element_self_close; -pub use element_close_end::ElementCloseEnd; pub use element_close_name::ElementCloseName; pub use element_close_start::ElementCloseStart; pub use element_open_name::ElementOpenName; @@ -18,12 +18,13 @@ pub use element_open_start::ElementOpenStart; pub use end_of_file::EndOfFile; pub use start_of_file::StartOfFile; pub use text_content::TextContent; -pub use whitespace::Whitespace; +pub use attribute_name::AttributeName; +pub use attribute_value::AttributeValue; +pub use element_self_close::ElementSelfClose; /// Represents the different states of XML token parsing #[derive(Clone, Debug)] pub enum Token { - ElementCloseEnd(ElementCloseEnd), ElementCloseName(ElementCloseName), ElementCloseStart(ElementCloseStart), ElementName(ElementOpenName), @@ -32,13 +33,14 @@ pub enum Token { EndOfFile(EndOfFile), StartOfFile(StartOfFile), TextContent(TextContent), - Whitespace(Whitespace), + AttributeName(AttributeName), + AttributeValue(AttributeValue), + ElementSelfClose(ElementSelfClose), } impl Token { pub fn append(self, c: char) -> Vec { match self { - Token::ElementCloseEnd(element_close_end) => element_close_end.append(c), Token::ElementCloseName(element_close_name) => element_close_name.append(c), Token::ElementCloseStart(element_close_start) => element_close_start.append(c), Token::ElementName(element_name) => element_name.append(c), @@ -47,7 +49,9 @@ impl Token { Token::EndOfFile(end_of_file) => end_of_file.append(c), Token::StartOfFile(start_of_file) => start_of_file.append(c), Token::TextContent(text_content) => text_content.append(c), - Token::Whitespace(whitespace) => whitespace.append(c), + Token::AttributeName(attribute_name) => attribute_name.append(c), + Token::AttributeValue(attribute_value) => attribute_value.append(c), + Token::ElementSelfClose(element_self_close) => element_self_close.append(c), } } @@ -57,4 +61,232 @@ impl Token { _ => false, } } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_start_of_file_transition() { + let element = create_test_element(); + let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element))); + + // Should only accept '<' + let next_tokens = token.append('<'); + assert_eq!(next_tokens.len(), 1, "Should have one transition"); + + match &next_tokens[0] { + Token::ElementStart(_) => (), + _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]), + } + + // Should reject other characters + let token = Token::StartOfFile(StartOfFile::new(Arc::clone(&element))); + let next_tokens = token.append('a'); + assert_eq!(next_tokens.len(), 0, "Should reject 'a'"); + } + + #[test] + fn test_element_start_transition() { + let element = create_test_element(); + let token = Token::ElementStart(ElementOpenStart::new(Arc::clone(&element))); + + // Should accept first char of element name + let next_tokens = token.append('r'); + assert_eq!(next_tokens.len(), 1, "Should have one transition"); + + match &next_tokens[0] { + Token::ElementName(_) => (), + _ => panic!("Expected ElementName, got {:?}", next_tokens[0]), + } + + // Should reject invalid char + let token = Token::ElementStart(ElementOpenStart::new(Arc::clone(&element))); + let next_tokens = token.append('x'); + assert_eq!(next_tokens.len(), 0, "Should reject 'x'"); + } + + #[test] + fn test_element_name_transition() { + let element = create_test_element(); + let token = Token::ElementName(ElementOpenName::new(Arc::clone(&element), "r".to_string()).unwrap()); + + // Continue building name + let next_tokens = token.append('e'); + assert_eq!(next_tokens.len(), 1, "Should have one transition"); + + match &next_tokens[0] { + Token::ElementName(_) => (), + _ => panic!("Expected ElementName, got {:?}", next_tokens[0]), + } + + // Build complete name and end with > + // Remove the unused buffer + let mut current_tokens = vec![Token::ElementStart(ElementOpenStart::new(Arc::clone(&element)))]; + + for c in "reasoning>".chars() { + let mut new_tokens = Vec::new(); + for token in current_tokens { + new_tokens.append(&mut token.append(c)); + } + current_tokens = new_tokens; + + assert!(!current_tokens.is_empty(), + "Should have valid transition after appending '{}'", c); + } + + // Check final state + assert_eq!(current_tokens.len(), 1, "Should have one final transition"); + match ¤t_tokens[0] { + Token::ElementOpen(_) => (), + _ => panic!("Expected ElementOpen, got {:?}", current_tokens[0]), + } + } + + #[test] + fn test_element_open_end_transition() { + let element = create_test_element(); + let token = Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&element))); + + // Should accept text content in mixed element + let next_tokens = token.append('t'); + assert_eq!(next_tokens.len(), 1, "Should accept text"); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + + // Should accept closing tag start + let token = Token::ElementOpen(ElementOpenEnd::new(Arc::clone(&element))); + let next_tokens = token.append('<'); + assert_eq!(next_tokens.len(), 1, "Should accept '<'"); + + match &next_tokens[0] { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_text_content_transition() { + let element = create_test_element(); + let token = Token::TextContent(TextContent::new(Arc::clone(&element))); + + // Should accept more text + let next_tokens = token.append('a'); + assert_eq!(next_tokens.len(), 1, "Should accept more text"); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + + // Should accept closing tag start + let token = Token::TextContent(TextContent::new(Arc::clone(&element))); + let next_tokens = token.append('<'); + assert_eq!(next_tokens.len(), 1, "Should accept '<'"); + + match &next_tokens[0] { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_element_close_start_transition() { + let element = create_test_element(); + let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element))); + + // Should accept / + let next_tokens = token.append('/'); + assert_eq!(next_tokens.len(), 1, "Should accept '/'"); + + let token = next_tokens[0].clone(); + match token { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart with slash, got {:?}", token), + } + + // After /, should accept first char of element name + let next_tokens = token.append('r'); + assert_eq!(next_tokens.len(), 1, "Should accept 'r'"); + + match &next_tokens[0] { + Token::ElementCloseName(_) => (), + _ => panic!("Expected ElementCloseName, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_element_close_name_transition() { + let element = create_test_element(); + let token = Token::ElementCloseStart(ElementCloseStart::new(Arc::clone(&element))); + + // Add / first + let next_tokens = token.append('/'); + assert_eq!(next_tokens.len(), 1, "Should accept '/'"); + + // Start building name - clone the token to avoid moving out of the vector + let next_tokens = next_tokens[0].clone().append('r'); + assert_eq!(next_tokens.len(), 1, "Should accept 'r'"); + + // Continue with each character + let mut current_tokens = next_tokens; + for c in "easoning".chars() { + let mut new_tokens = Vec::new(); + for token in current_tokens { + new_tokens.append(&mut token.append(c)); + } + current_tokens = new_tokens; + + assert!(!current_tokens.is_empty(), + "Should have valid transition after appending '{}'", c); + } + + // Final > to close - clone the token to avoid moving out of the vector + let next_tokens = current_tokens[0].clone().append('>'); + assert_eq!(next_tokens.len(), 1, "Should accept '>'"); + + match &next_tokens[0] { + Token::EndOfFile(_) => (), + _ => panic!("Expected EndOfFile, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_end_of_file_properties() { + let token = Token::EndOfFile(EndOfFile::new()); + + // Should be identified as EOF + assert!(token.is_eof(), "EndOfFile should be recognized as EOF"); + + // Should not accept any more input + let next_tokens = token.append('a'); + assert_eq!(next_tokens.len(), 0, "Should not accept any more input"); + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/start_of_file.rs b/lib/xml_schema_validator/src/token/start_of_file.rs index 27e3232..db47fd9 100644 --- a/lib/xml_schema_validator/src/token/start_of_file.rs +++ b/lib/xml_schema_validator/src/token/start_of_file.rs @@ -19,4 +19,57 @@ impl StartOfFile { vec![] } } +} + +#[cfg(test)] +mod start_of_file_tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_start_of_file_less_than() { + let element = create_test_element(); + let token = StartOfFile::new(Arc::clone(&element)); + + // Test with < + let next_tokens = token.append('<'); + assert!(!next_tokens.is_empty(), "Should accept '<'"); + + match &next_tokens[0] { + Token::ElementStart(_) => (), + _ => panic!("Expected ElementStart, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_start_of_file_invalid() { + let element = create_test_element(); + let token = StartOfFile::new(Arc::clone(&element)); + + // Test with invalid character + let next_tokens = token.append('a'); + assert!(next_tokens.is_empty(), "Should reject invalid character"); + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/text_content.rs b/lib/xml_schema_validator/src/token/text_content.rs index 3434385..c22f0f9 100644 --- a/lib/xml_schema_validator/src/token/text_content.rs +++ b/lib/xml_schema_validator/src/token/text_content.rs @@ -23,4 +23,62 @@ impl TextContent { ))] } } +} + +#[cfg(test)] +mod text_content_tests { + use super::*; + use std::sync::Arc; + use crate::schema; + + fn create_test_element() -> Arc { + let element = schema::XsElement { + name: "reasoning".to_string(), + text: None, + xs_complex_type: schema::XsComplexType { + mixed: Some("true".to_string()), + text: None, + xs_attribute: None, + xs_sequence: Some(schema::XsSequence { + text: None, + xs_any: schema::XsAny { + min_occurs: "0".to_string(), + max_occurs: "unbounded".to_string(), + process_contents: "skip".to_string(), + }, + }), + }, + }; + Arc::new(element) + } + + #[test] + fn test_text_content_normal_char() { + let element = create_test_element(); + let token = TextContent::new(Arc::clone(&element)); + + // Test with normal character + let next_tokens = token.append('a'); + assert!(!next_tokens.is_empty(), "Should accept normal character"); + + match &next_tokens[0] { + Token::TextContent(_) => (), + _ => panic!("Expected TextContent, got {:?}", next_tokens[0]), + } + } + + #[test] + fn test_text_content_less_than() { + let element = create_test_element(); + let token = TextContent::new(Arc::clone(&element)); + + // Test with < (start of tag) + let next_tokens = token.append('<'); + assert!(!next_tokens.is_empty(), "Should accept '<'"); + + match &next_tokens[0] { + Token::ElementCloseStart(_) => (), + _ => panic!("Expected ElementCloseStart, got {:?}", next_tokens[0]), + } + } } \ No newline at end of file diff --git a/lib/xml_schema_validator/src/token/whitespace.rs b/lib/xml_schema_validator/src/token/whitespace.rs deleted file mode 100644 index 71f432c..0000000 --- a/lib/xml_schema_validator/src/token/whitespace.rs +++ /dev/null @@ -1,29 +0,0 @@ -use crate::*; - -/// Represents one or more whitespace characters -#[derive(Clone, Debug)] -pub struct Whitespace { - next: Vec -} - -impl Whitespace { - pub fn new(next: Vec) -> Self { - Self { - next - } - } - - pub fn append(self, c: char) -> Vec { - if c.is_whitespace() { - vec![Token::Whitespace(Self { - next: self.next - })] - } else { - let mut tokens = vec![]; - for token in self.next { - tokens.append(&mut token.append(c)) - } - tokens - } - } -} \ No newline at end of file diff --git a/lib/xml_schema_validator/tests/test_logits_processor.py b/lib/xml_schema_validator/tests/test_logits_processor.py index 327179c..57b586a 100644 --- a/lib/xml_schema_validator/tests/test_logits_processor.py +++ b/lib/xml_schema_validator/tests/test_logits_processor.py @@ -1,14 +1,14 @@ -from xml_logits_processor import XmlLogitsProcessor -from threading import Thread -from transformers import pipeline, AutoTokenizer, TextIteratorStreamer +from transformers import AutoTokenizer +from xml_schema_validator import XmlLogitsProcessor import os -import sys +import time +import torch -def test_logits_processor(): - print("test_logits_processor") +model_path = "../../model" +api_token = os.environ["SIA_HF_API_KEY"] - model_name = "../../model" - xml_schema_text = """ +xml_schema_actions = open("../../action_schema.xsd").read() +xml_schema_only_root_node = """ @@ -19,53 +19,106 @@ def test_logits_processor(): """ - model = "google/gemma-3-1b-it" +def test_logits_processor_init_copy(): + """Test the initialization of the LogitsProcessor""" - pipline = pipeline( - "text-generation", - model=model, - token=os.environ["SIA_HF_API_KEY"], - ) + tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) + logits_processor = XmlLogitsProcessor(tokenizer, xml_schema_only_root_node) + assert logits_processor.core is not None - tokenizer = AutoTokenizer.from_pretrained( - model, - token=os.environ["SIA_HF_API_KEY"], - ) + logits_processor_copy = logits_processor.copy() + assert logits_processor_copy.core is not None - logits_processor = XmlLogitsProcessor(tokenizer, xml_schema_text) - - messages = [ - {"role": "system", "content": "Always respond with message by the user. So if the user says 'hello world', you response hello world"}, - {"role": "user", "content": "hello, I am the user"} - ] +def test_xml_schema_parsing(): + """Test basic XML schema parsing with different element types""" + + tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) + logits_processor = XmlLogitsProcessor(tokenizer, xml_schema_only_root_node) + assert logits_processor.core is not None - text_inputs = tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True - ) + element_names = logits_processor.core.get_element_names() + assert len(element_names) > 0 + assert "root" in element_names - streamer = TextIteratorStreamer( - tokenizer, - skip_prompt=True - ) +def test_basic_xml_validation(): + """Test basic validation of XML fragments against the schema""" - generation_kwargs = { - "text_inputs": text_inputs, - "max_new_tokens": 20, - "streamer": streamer, - #"logits_processor": [logits_processor], - } + tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) + processor = XmlLogitsProcessor(tokenizer, xml_schema_actions) + + # Test appending valid XML fragments + assert processor.core.copy().append("") + assert processor.core.copy().append("Test content") + + # Test appending invalid XML fragments + assert not processor.core.copy().append("") + assert not processor.core.copy().append("") - generation_thread = Thread( - target=pipline, - kwargs=generation_kwargs - ) +def test_token_masking(): + """Test that invalid tokens are properly masked""" + + tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) + processor = XmlLogitsProcessor(tokenizer, xml_schema_actions) + + # Create dummy input_ids and scores + input_ids = torch.tensor([tokenizer.encode("")]) + scores = torch.ones((1, len(tokenizer))) # All tokens have equal probability + + # Process the scores + processed_scores = processor(input_ids, scores) + + # Verify that valid continuations still have positive scores + assert processed_scores[0, tokenizer.encode(" ", add_special_tokens=False)[0]] > -float('inf') + + # Verify that invalid continuations are masked + assert processed_scores[0, tokenizer.encode("", add_special_tokens=False)[0]] == -float('inf') - generation_thread.start() - - for text in streamer: - print(text, end="", file=sys.stderr, flush=True) - - generation_thread.join() +def test_performance(): + """Test performance with larger XML documents""" + + tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) + processor = XmlLogitsProcessor(tokenizer, xml_schema_actions) + + # Generate a larger XML document + large_xml = "" + "Test content. " * 100 + "" + + # Measure time to process + start_time = time.time() + processor.core.append(large_xml) + processing_time = time.time() - start_time + + print(f"Processing time for large XML: {processing_time:.4f} seconds") + # You might want to assert that processing time is below a threshold +def test_subword_tokens(): + """Test handling of subword tokens that might split XML tags""" + tokenizer = AutoTokenizer.from_pretrained(model_path, token=api_token) + schema_text = open("../../action_schema.xsd").read() + + processor = XmlLogitsProcessor(tokenizer, schema_text) + + # Find some XML tag that gets split into multiple tokens by your tokenizer + tag = "" + tokens = tokenizer.encode(tag, add_special_tokens=False) + + if len(tokens) > 1: + print(f"Tag '{tag}' is split into {len(tokens)} tokens") + + # Test that the processor can handle these split tokens + import torch + input_ids = torch.tensor([[tokens[0]]]) + scores = torch.ones((1, len(tokenizer))) + + processed_scores = processor(input_ids, scores) + + # The next token in the sequence should have a high score + assert processed_scores[0, tokens[1]] > -float('inf') + if __name__ == "__main__": - test_logits_processor() \ No newline at end of file + # Run individual tests for debugging + test_logits_processor_init_copy() + test_xml_schema_parsing() + test_basic_xml_validation() + test_token_masking() + test_performance() + test_subword_tokens() diff --git a/scripts/collect.sh b/scripts/collect.sh index 5161679..9f753bd 100755 --- a/scripts/collect.sh +++ b/scripts/collect.sh @@ -162,16 +162,6 @@ process_command() { output_abs_path=$(realpath -m "$output_file") output_rel_path=$(realpath --relative-to=. "$output_abs_path") - # Generate directory tree - { - echo "Directory Tree:" - if command -v tree &>/dev/null; then - tree -a -I '.git' . - else - find . -name .git -prune -o -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g' - fi - } > "$output_file" - # Process directories and collect files declare -a all_files=() @@ -262,7 +252,7 @@ process_command() { # Append file contents to output { - echo -e "\n\nFile Contents:" + echo -e "File Contents:" for file in "${unique_files[@]}"; do if [[ -L "$file" ]]; then target=$(readlink -f "$file" || echo "unknown") @@ -272,7 +262,7 @@ process_command() { cat "$file" 2>/dev/null || echo "Error: Unable to read file" fi done - } >> "$output_file" + } > "$output_file" fi }