Tool Evaluation
Tool Evaluation
Multiple agents independently run a single evaluation task from an evaluation file.
import json
import re
import time
import traceback
import xml.etree.ElementTree as ET # noqa: S314
from pathlib import Path
from typing import Any
from anthropic import Anthropic
Prompts
Prompts
# Embedded evaluator prompt
EVALUATION_PROMPT = """You are an AI assistant with access to tools.
When given a task, you MUST:
1. Use the available tools to complete the task
2. Provide summary of each step in your approach, wrapped in <summary> tags
3. Provide feedback on the tools provided, wrapped in <feedback> tags
4. Provide your final response, wrapped in <response> tags
Summary Requirements:
- In your <summary> tags, you must explain:
- The steps you took to complete the task
- Which tools you used, in what order, and why
- The inputs you provided to each tool
- The outputs you received from each tool
- A summary for how you arrived at the response
Feedback Requirements:
- In your <feedback> tags, provide constructive feedback on the tools:
- Comment on tool names: Are they clear and descriptive?
- Comment on input parameters: Are they well-documented? Are required vs optional parameters clear?
- Comment on descriptions: Do they accurately describe what the tool does?
- Comment on any errors encountered during tool usage: Did the tool fail to execute? Did the tool return too many tokens?
- Identify specific areas for improvement and explain WHY they would help
- Be specific and actionable in your suggestions
Response Requirements:
- Your response should be concise and directly address what was asked
- Always wrap your final response in <response> tags
- If you cannot solve the task return <response>NOT_FOUND</response>
- For numeric responses, provide just the number
- For IDs, provide just the ID
- For names or text, provide the exact text requested
- Your response should go last"""
Agent Loop
Agent Loop
client = Anthropic()
model = "claude-sonnet-5-5"
def agent_loop(prompt: str, tools: list[dict[str, Any]] = None) -> tuple[str, dict[str, Any]]:
"""Simplified agent class for tool evaluation"""
messages = [{"role": "user", "content": prompt}]
response = client.messages.create(
model=model,
max_tokens=4096,
system=EVALUATION_PROMPT,
messages=messages,
tools=tools,
)
messages.append({"role": "assistant", "content": response.content})
# Track tool calls with timing
tool_metrics = {} # {tool_name: {"count": N, "durations": [X1, X2, ...]}}
def _prepare_tool_result(tool_use_id, tool_result):
return {
"type": "tool_result",
"tool_use_id": tool_use_id,
"content": tool_result,
}
while response.stop_reason == "tool_use":
# The model may call several tools in one turn; every tool_use needs a tool_result
tool_results = []
for tool_use in (block for block in response.content if block.type == "tool_use"):
tool_name = tool_use.name
tool_start_ts = time.time()
try:
# Note: eval is used here for demonstration purposes to dynamically call tool functions.
# In production, use a safer dispatch mechanism like a dictionary of functions.
tool_response = eval( # noqa: S307
f"{tool_name}(**tool_use.input)"
) # Call the tool function with its input
except Exception as e:
tool_response = f"Error executing tool {tool_name}: {str(e)}\n"
tool_response += traceback.format_exc()
tool_duration = time.time() - tool_start_ts
# Update tool metrics
if tool_name not in tool_metrics:
tool_metrics[tool_name] = {"count": 0, "durations": []}
tool_metrics[tool_name]["count"] += 1
tool_metrics[tool_name]["durations"].append(tool_duration)
tool_results.append(_prepare_tool_result(tool_use.id, tool_response))
# Append all tool results in a single user message
messages.append({"role": "user", "content": tool_results})
response = client.messages.create(
model=model,
max_tokens=4096,
system=EVALUATION_PROMPT,
messages=messages,
tools=tools,
)
messages.append({"role": "assistant", "content": response.content})
response = next(
(block.text for block in response.content if hasattr(block, "text")),
None,
)
return response, tool_metrics
Helper Functions
Helper Functions
def parse_evaluation_file(file_path: Path) -> list[dict[str, Any]]:
"""Parse XML evaluation file and return list of evaluation tasks."""
try:
# Parse trusted local XML evaluation file
tree = ET.parse(file_path) # noqa: S314
root = tree.getroot()
evaluations = []
# Check for task elements
tasks = root.findall(".//task")
for task in tasks:
prompt_elem = task.find("prompt")
response_elem = task.find("response")
if prompt_elem is not None and response_elem is not None:
eval_dict = {
"prompt": (prompt_elem.text or "").strip(),
"response": (response_elem.text or "").strip(),
}
evaluations.append(eval_dict)
return evaluations
except Exception as e:
print(f"Error parsing evaluation file {file_path}: {e}")
return []
def evaluate_single_task(
task: dict[str, Any], tools: list[dict[str, Any]], task_index: int
) -> dict[str, Any]:
"""Evaluate a single task with the given tools."""
start_time = time.time()
# Run the task
print(f"Task {task_index + 1}: Running task with prompt: {task['prompt']}")
response, tool_metrics = agent_loop(task["prompt"], tools)
# Extract all tagged content
def _extract_xml_content(text, tag):
pattern = rf"<{tag}>(.*?)</{tag}>"
matches = re.findall(pattern, text, re.DOTALL)
return matches[-1].strip() if matches else None
response, summary, feedback = (
_extract_xml_content(response, tag) for tag in ["response", "summary", "feedback"]
)
duration_seconds = time.time() - start_time
return {
"prompt": task["prompt"],
"expected": task["response"],
"actual": response,
"score": int(response == task["response"]),
"total_duration": duration_seconds,
"tool_calls": tool_metrics,
"num_tool_calls": sum(len(metrics["durations"]) for metrics in tool_metrics.values()),
"summary": summary,
"feedback": feedback,
}
Main Evaluation Function
Main Evaluation Function
# Report Templates
REPORT_HEADER = """
# Evaluation Report
## Summary
- **Accuracy**: {correct}/{total} ({accuracy:.1f}%)
- **Average Task Duration**: {average_duration_s:.2f}s
- **Average Tool Calls per Task**: {average_tool_calls:.2f}
- **Total Tool Calls**: {total_tool_calls}
---
"""
TASK_TEMPLATE = """
### Task
**Prompt**: {prompt}
**Ground Truth Response**: `{expected_response}`
**Actual Response**: `{actual_response}`
**Correct**: {correct_indicator}
**Duration**: {total_duration:.2f}s
**Tool Calls**: {tool_calls}
**Summary**
{summary}
**Feedback**
{feedback}
---
"""
def run_evaluation(eval_path: str, tools: list[dict[str, Any]]) -> str:
"""
Run evaluation with provided tools using a simple loop.
Args:
eval_path: Path to XML evaluation file
tools: List of tool definitions to use for evaluation
"""
print("🚀 Starting Evaluation")
eval_file = Path(eval_path)
# Parse evaluation tasks
tasks = parse_evaluation_file(eval_file)
print(f"📋 Loaded {len(tasks)} evaluation tasks")
# Simple loop to run all tasks
results = []
for i, task in enumerate(tasks):
print(f"Processing task {i + 1}/{len(tasks)}")
results.append(evaluate_single_task(task, tools, i))
# Calculate summary statistics
correct = sum(r["score"] for r in results)
accuracy = (correct / len(results)) * 100
average_duration_s = sum(r["total_duration"] for r in results) / len(results)
average_tool_calls = sum(r["num_tool_calls"] for r in results) / len(results)
total_tool_calls = sum(r["num_tool_calls"] for r in results)
report = REPORT_HEADER.format(
correct=correct,
total=len(results),
accuracy=accuracy,
average_duration_s=average_duration_s,
average_tool_calls=average_tool_calls,
total_tool_calls=total_tool_calls,
)
report += "".join(
[
TASK_TEMPLATE.format(
prompt=task["prompt"],
expected_response=task["response"],
actual_response=result["actual"],
correct_indicator="✅" if result["score"] else "❌",
total_duration=result["total_duration"],
tool_calls=json.dumps(result["tool_calls"], indent=2),
summary=result["summary"] or "N/A",
feedback=result["feedback"] or "N/A",
)
for task, result in zip(tasks, results, strict=False)
]
)
# Join all sections into final report
return report
Calculator Tool
Calculator Tool
def calculator(expression: str) -> str:
"""A basic calculator that performs arithmetic operations."""
try:
# Note: eval with restricted builtins is used here for demonstration.
# In production, use a safer alternative like a math expression parser.
result = eval(expression, {"__builtins__": {}}, {}) # noqa: S307
return str(result)
except Exception as e:
return f"Error: {str(e)}"
# Define the tool schema for the calculator
calculator_tool = {
"name": "calculator",
"description": "", # An unhelpful tool description.
"input_schema": {
"type": "object",
"properties": {
"expression": {
"type": "string",
"description": "", # An unhelpful schema description.
}
},
"required": ["expression"],
},
}
# Set the tools list
tools = [calculator_tool]
Run Evaluation
Run Evaluation
# Run evaluation
print("✅ Using calculator tool")
report = run_evaluation(eval_path="evaluation.xml", tools=tools)
print(report)
✅ Using calculator tool
🚀 Starting Evaluation
📋 Loaded 8 evaluation tasks
Processing task 1/8
Task 1: Running task with prompt: Calculate the compound interest on $10,000 invested at 5% annual interest rate, compounded monthly for 3 years. What is the final amount in dollars (rounded to 2 decimal places)?
Processing task 2/8
Task 2: Running task with prompt: A projectile is launched at a 45-degree angle with an initial velocity of 50 m/s. Calculate the total distance (in meters) it has traveled from the launch point after 2 seconds, assuming g=9.8 m/s². Round to 2 decimal places.
Processing task 3/8
Task 3: Running task with prompt: A sphere has a volume of 500 cubic meters. Calculate its surface area in square meters. Round to 2 decimal places.
Processing task 4/8
Task 4: Running task with prompt: Calculate the population standard deviation of this dataset: [12, 15, 18, 22, 25, 30, 35]. Round to 2 decimal places.
Processing task 5/8
Task 5: Running task with prompt: Calculate the pH of a solution with a hydrogen ion concentration of 3.5 × 10^-5 M. Round to 2 decimal places.
Processing task 6/8
Task 6: Running task with prompt: Calculate the monthly payment for a $200,000 mortgage at 4.5% annual interest rate for 30 years (360 months). Use the standard mortgage payment formula. Round to 2 decimal places.
Processing task 7/8
Task 7: Running task with prompt: Calculate the energy in joules of a photon with wavelength 550 nanometers. Use h = 6.626 × 10^-34 J·s and c = 3 × 10^8 m/s. Express the answer in scientific notation with 2 significant figures after the decimal (e.g., 3.61e-19).
Processing task 8/8
Task 8: Running task with prompt: Find the larger real root of the quadratic equation 3x² - 7x + 2 = 0. Give the exact value.
# Evaluation Report
## Summary
- **Accuracy**: 8/8 (100.0%)
- **Average Task Duration**: 6.43s
- **Average Tool Calls per Task**: 2.75
- **Total Tool Calls**: 22
---
### Task
**Prompt**: Calculate the compound interest on $10,000 invested at 5% annual interest rate, compounded monthly for 3 years. What is the final amount in dollars (rounded to 2 decimal places)?
**Ground Truth Response**: `11614.72`
**Actual Response**: `11614.72`
**Correct**: ✅
**Duration**: 6.71s
**Tool Calls**: {
"calculator": {
"count": 3,
"durations": [
9.393692016601562e-05,
0.00010085105895996094,
8.034706115722656e-05
]
}
}
**Summary**
1. I called the calculator with `10000 * (1 + 0.05/12)^(36)`. It returned an error because the `^` operator isn't supported for floats.
2. I called it again with `round(10000 * (1 + 0.05/12)**36, 2)`. It returned an error because `round` isn't defined.
3. I called it a third time with `10000 * (1 + 0.05/12)**36`. It returned 11614.722313334678.
4. I rounded that result to 2 decimal places by hand, which gives 11614.72.
**Feedback**
The calculator tool has a clear name, but its description is missing. It doesn't say which operators or functions it supports.
- The `^` operator fails with a confusing Python type error. `**` works. Documenting this, or treating `^` as exponentiation, would help.
- `round` is not available. Either support it or state that rounding must be done by hand.
- The `expression` parameter has no description. It should list the supported syntax, for example "Python arithmetic expression; use ** for powers".
- The output has many unformatted decimal places. An optional precision parameter would help.
---
### Task
**Prompt**: A projectile is launched at a 45-degree angle with an initial velocity of 50 m/s. Calculate the total distance (in meters) it has traveled from the launch point after 2 seconds, assuming g=9.8 m/s². Round to 2 decimal places.
**Ground Truth Response**: `87.25`
**Actual Response**: `87.25`
**Correct**: ✅
**Duration**: 6.49s
**Tool Calls**: {
"calculator": {
"count": 2,
"durations": [
0.00010132789611816406,
9.870529174804688e-05
]
}
}
**Summary**
I took "distance from the launch point" to mean the straight-line distance between the launch point and the projectile's position at t = 2 s. This is the magnitude of the displacement, not the length of the curved path.
1. I first called the calculator with sqrt and trig functions. It returned the error "name 'sqrt' is not defined".
2. I called it again using `**0.5` for the square root and `2**0.5/2` for cos 45° and sin 45°.
- Input: `((50*2**0.5/2*2)**2 + (50*2**0.5/2*2 - 0.5*9.8*4)**2)**0.5`
- Output: 87.24850381381181
The components at t = 2 s are:
- x = 50·cos45°·2 = 70.71 m
- y = 50·sin45°·2 − ½·9.8·2² = 70.71 − 19.6 = 51.11 m
The distance is √(70.71² + 51.11²) ≈ 87.25 m.
**Feedback**
The calculator tool is named clearly. Its description and parameter documentation don't say which functions are supported. My first call failed because `sqrt` is undefined, and the docs gave no hint of that. Please list the supported functions and constants (sqrt, sin, cos, pi) and the syntax, for example that Python-style `**` is expected. The error message could also suggest alternatives, such as "use **0.5".
---
### Task
**Prompt**: A sphere has a volume of 500 cubic meters. Calculate its surface area in square meters. Round to 2 decimal places.
**Ground Truth Response**: `304.65`
**Actual Response**: `304.65`
**Correct**: ✅
**Duration**: 5.69s
**Tool Calls**: {
"calculator": {
"count": 2,
"durations": [
9.560585021972656e-05,
8.678436279296875e-05
]
}
}
**Summary**
I first called the calculator with the expression 4*pi*(3*500/(4*pi))^(2/3). It returned the error "name 'pi' is not defined". I then retried with a numeric value of pi (3.141592653589793) and ** for the exponent. That call returned 304.6473892689777.
The calculation uses V = (4/3)πr³, so r = (3V/(4π))^(1/3). The surface area is A = 4πr² = 4π(3V/(4π))^(2/3). With V = 500, this gives about 304.65 m².
**Feedback**
The calculator tool's name is clear. The description and parameter docs are missing, though. It does not say which syntax is supported (for example, whether ^ or ** is used for powers) or which constants are defined. The `pi` constant was not recognized, which caused an error. Supporting `pi` and `e`, or listing the supported constants and functions in the description, would help. The "expression" parameter would also benefit from an example.
---
### Task
**Prompt**: Calculate the population standard deviation of this dataset: [12, 15, 18, 22, 25, 30, 35]. Round to 2 decimal places.
**Ground Truth Response**: `7.61`
**Actual Response**: `7.61`
**Correct**: ✅
**Duration**: 5.33s
**Tool Calls**: {
"calculator": {
"count": 2,
"durations": [
0.00011467933654785156,
0.00010204315185546875
]
}
}
**Summary**
Steps: I called the calculator to compute the population standard deviation. The mean is 157/7 ≈ 22.43.
1. First call: the expression used `sqrt(...)` with the mean written as a decimal. It returned the error "name 'sqrt' is not defined".
2. Second call: I replaced sqrt with `**0.5` and used the exact mean 157/7. The expression was (sum of squared deviations / 7)**0.5. It returned 7.61309291112813.
Rounded to 2 decimal places, that is 7.61.
**Feedback**
The tool is named clearly, but its description doesn't say which functions and syntax it supports. The `sqrt` call failed with an unhelpful error. Listing the supported functions (e.g. sqrt, pow) or the allowed syntax (`**`) would avoid that retry. Adding built-in statistical functions such as mean and stdev would also help. The `expression` parameter has no description or example.
---
### Task
**Prompt**: Calculate the pH of a solution with a hydrogen ion concentration of 3.5 × 10^-5 M. Round to 2 decimal places.
**Ground Truth Response**: `4.46`
**Actual Response**: `4.46`
**Correct**: ✅
**Duration**: 14.60s
**Tool Calls**: {
"calculator": {
"count": 8,
"durations": [
6.961822509765625e-05,
7.319450378417969e-05,
2.4557113647460938e-05,
8.20159912109375e-05,
2.5272369384765625e-05,
2.3126602172851562e-05,
0.00012564659118652344,
0.00013303756713867188
]
}
}
**Summary**
I needed pH = -log10[H⁺] with [H⁺] = 3.5 × 10⁻⁵ M.
1. I called the calculator with `-log10(3.5e-5)`. It returned an error because `log10` isn't defined.
2. I tried `math.log10`, `log(x, 10)`, `np.log10`, `ln`, and `__import__('math')`. Each one failed with a "name not defined" error.
3. I then rewrote the problem as pH = 5 − log10(3.5). I computed ln(3.5) with the atanh series 2·Σ x^(2k+1)/(2k+1), where x = 2.5/4.5 = 5/9. I divided by ln(10) = 2.302585093.
4. A first attempt using `sum` and `range` failed because `sum` isn't defined. Writing out the series terms explicitly worked and returned 4.45593.
5. Rounded to 2 decimal places, that is 4.46.
**Feedback**
The calculator tool has no documentation of which functions it supports. Every standard logarithm form I tried failed (`log10`, `math.log10`, `np.log10`, `log`, `ln`), and so did `sum` and `__import__`. The name "calculator" suggests it would handle basic scientific functions.
- Improvement: list the supported functions and syntax in the description, or add the common ones (log, log10, ln, exp, sqrt, and so on). Chemistry and science tasks need logarithms often.
- Improvement: make the error messages more helpful. For example, "name 'log10' is not defined. Available functions: ..." would save a lot of trial and error.
- The `expression` parameter is a bare string with no description. A note on the expected syntax would help.
---
### Task
**Prompt**: Calculate the monthly payment for a $200,000 mortgage at 4.5% annual interest rate for 30 years (360 months). Use the standard mortgage payment formula. Round to 2 decimal places.
**Ground Truth Response**: `1013.37`
**Actual Response**: `1013.37`
**Correct**: ✅
**Duration**: 4.65s
**Tool Calls**: {
"calculator": {
"count": 2,
"durations": [
9.012222290039062e-05,
8.034706115722656e-05
]
}
}
**Summary**
1. I called the calculator with the standard mortgage formula, M = P·r·(1+r)^n / ((1+r)^n − 1). The inputs were P = 200000, r = 0.045/12 and n = 360. I wrapped the expression in `round(..., 2)`. The tool returned the error "name 'round' is not defined".
2. I ran the same expression again without `round`. It returned 1013.3706196517716.
3. I rounded that result to 2 decimal places by hand, which gives 1013.37.
**Feedback**
The tool name "calculator" is clear. The parameter description for `expression` is empty. It doesn't say which functions or syntax are supported, such as `round`, `**` or `^`. I only found out that `round` isn't supported by trying it and getting an error. Please document the supported operators and functions in the description. Adding common functions like `round`, `abs` and `sqrt` would also help, since rounding is a frequent need.
---
### Task
**Prompt**: Calculate the energy in joules of a photon with wavelength 550 nanometers. Use h = 6.626 × 10^-34 J·s and c = 3 × 10^8 m/s. Express the answer in scientific notation with 2 significant figures after the decimal (e.g., 3.61e-19).
**Ground Truth Response**: `3.61e-19`
**Actual Response**: `3.61e-19`
**Correct**: ✅
**Duration**: 2.77s
**Tool Calls**: {
"calculator": {
"count": 1,
"durations": [
7.081031799316406e-05
]
}
}
**Summary**
I used the calculator tool once. The input was the expression 6.626e-34 * 3e8 / 550e-9, which is E = hc/λ with λ = 550 nm = 550e-9 m. The output was 3.614181818181818e-19 J. Rounding to 2 digits after the decimal point gives 3.61e-19.
**Feedback**
The calculator tool worked without errors. Its name is clear. The description of the `expression` parameter is empty, so it doesn't say which syntax is supported (for example, whether scientific notation like 1e-9 is accepted). A short description with example expressions would help. The output was accurate, with full precision.
---
### Task
**Prompt**: Find the larger real root of the quadratic equation 3x² - 7x + 2 = 0. Give the exact value.
**Ground Truth Response**: `2`
**Actual Response**: `2`
**Correct**: ✅
**Duration**: 5.18s
**Tool Calls**: {
"calculator": {
"count": 2,
"durations": [
7.557868957519531e-05,
7.843971252441406e-05
]
}
}
**Summary**
I wanted to compute the larger root with the quadratic formula, x = (7 + √(49 − 24)) / 6. The discriminant is 49 − 4·3·2 = 25.
1. I called the calculator with "(7 + sqrt(49 - 24)) / 6". It returned the error "name 'sqrt' is not defined".
2. I called it again with "(7 + (49 - 24)**0.5) / 6". It returned 2.0.
The discriminant is a perfect square, so the roots are (7 ± 5)/6, which gives 2 and 1/3. The larger root is exactly 2. Factoring confirms this: 3x² − 7x + 2 = (3x − 1)(x − 2).
**Feedback**
The calculator tool's name is clear. The `expression` parameter has no description, so I couldn't tell which functions or syntax are supported. My first call failed because `sqrt` isn't defined, and I only found that out by trying it. The tool description should list the supported functions (for example sqrt, pow) and the expected syntax. Documenting the evaluator, such as Python-style with `**`, would also help. Supporting `sqrt` would be a useful addition.
---