157 lines
5.3 KiB
Python
157 lines
5.3 KiB
Python
"""
|
||
分析LLM调用记录
|
||
|
||
使用方法：
|
||
python tests/analyze_llm_calls.py tests/llm_calls_20251031_150543.json
|
||
"""
|
||
|
||
import sys
|
||
import json
|
||
|
||
|
||
def analyze_llm_calls(json_file):
|
||
"""分析LLM调用记录"""
|
||
with open(json_file, 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
|
||
print("\n" + "="*80)
|
||
print("LLM调用分析报告")
|
||
print("="*80)
|
||
|
||
print(f"\n总调用次数: {data['total_calls']}")
|
||
|
||
for i, call in enumerate(data['calls'], 1):
|
||
print(f"\n{'─'*80}")
|
||
print(f"调用 #{i}")
|
||
print('─'*80)
|
||
|
||
# 时间信息
|
||
start = call.get('timestamp_start', 'N/A')
|
||
end = call.get('timestamp_end', 'N/A')
|
||
print(f"时间: {start} -> {end}")
|
||
|
||
# 消息数
|
||
messages = call.get('messages', [[]])
|
||
if messages:
|
||
msg_count = len(messages[0])
|
||
print(f"输入消息数: {msg_count}")
|
||
|
||
# 显示最后一条消息类型
|
||
if messages[0]:
|
||
last_msg = messages[0][-1]
|
||
print(f"最后一条输入消息: {last_msg['type']}")
|
||
|
||
# 响应信息
|
||
response = call.get('response', {})
|
||
generations = response.get('generations', [])
|
||
|
||
if generations:
|
||
gen = generations[0]
|
||
msg = gen.get('message', {})
|
||
|
||
print(f"响应类型: {msg.get('type', 'N/A')}")
|
||
|
||
# 内容
|
||
content = msg.get('content', '')
|
||
if content:
|
||
preview = content[:100].replace('\n', ' ')
|
||
print(f"响应内容: {preview}...")
|
||
|
||
# 工具调用
|
||
tool_calls = msg.get('tool_calls', [])
|
||
if tool_calls:
|
||
print(f"工具调用: {len(tool_calls)} 个")
|
||
for tc in tool_calls:
|
||
print(f" - {tc['name']}")
|
||
else:
|
||
print("工具调用: 无")
|
||
|
||
# Token使用
|
||
llm_output = response.get('llm_output', {})
|
||
token_usage = llm_output.get('token_usage', {})
|
||
if token_usage:
|
||
print(f"Token使用: {token_usage.get('prompt_tokens', 0)} input + {token_usage.get('completion_tokens', 0)} output = {token_usage.get('total_tokens', 0)} total")
|
||
|
||
print("\n" + "="*80)
|
||
print("执行流程总结")
|
||
print("="*80)
|
||
|
||
# 分析执行流程
|
||
call_summaries = []
|
||
for i, call in enumerate(data['calls'], 1):
|
||
response = call.get('response', {})
|
||
generations = response.get('generations', [])
|
||
|
||
if generations:
|
||
msg = generations[0].get('message', {})
|
||
tool_calls = msg.get('tool_calls', [])
|
||
|
||
if tool_calls:
|
||
tools = [tc['name'] for tc in tool_calls]
|
||
call_summaries.append(f"调用#{i}: {', '.join(tools)}")
|
||
else:
|
||
content_preview = msg.get('content', '')[:50].replace('\n', ' ')
|
||
call_summaries.append(f"调用#{i}: 返回文本 ({content_preview}...)")
|
||
|
||
for summary in call_summaries:
|
||
print(f" {summary}")
|
||
|
||
# 判断是否完成
|
||
print("\n" + "="*80)
|
||
print("状态判断")
|
||
print("="*80)
|
||
|
||
last_call = data['calls'][-1]
|
||
last_response = last_call.get('response', {})
|
||
last_generations = last_response.get('generations', [])
|
||
|
||
if last_generations:
|
||
last_msg = last_generations[0].get('message', {})
|
||
last_tool_calls = last_msg.get('tool_calls', [])
|
||
|
||
if not last_tool_calls:
|
||
print("⚠️ 最后一次调用没有工具调用")
|
||
print("原因: SubAgent返回了纯文本响应，导致主Agent停止")
|
||
print("影响: Agent停止执行，未完成完整流程")
|
||
print("\n预期行为: 主Agent应该继续执行步骤3（并行搜索）")
|
||
else:
|
||
print("✅ 最后一次调用有工具调用，流程继续")
|
||
else:
|
||
print("❌ 无法判断状态")
|
||
|
||
# 检查是否完成意图分析
|
||
search_queries_created = False
|
||
for call in data['calls']:
|
||
response = call.get('response', {})
|
||
generations = response.get('generations', [])
|
||
if generations:
|
||
msg = generations[0].get('message', {})
|
||
tool_calls = msg.get('tool_calls', [])
|
||
for tc in tool_calls:
|
||
if tc['name'] == 'write_file' and '/search_queries.json' in str(tc.get('args', {})):
|
||
search_queries_created = True
|
||
|
||
print("\n" + "="*80)
|
||
print("步骤完成情况")
|
||
print("="*80)
|
||
print(f"✅ 步骤1: 初始化 - 已完成 (/question.txt, /config.json)")
|
||
print(f"✅ 步骤2: 意图分析 - {'已完成' if search_queries_created else '未完成'} (/search_queries.json)")
|
||
print(f"❌ 步骤3: 并行搜索 - 未开始")
|
||
print(f"❌ 后续步骤 - 未开始")
|
||
|
||
print("\n" + "="*80)
|
||
print("建议")
|
||
print("="*80)
|
||
print("1. 问题根源: intent-analyzer SubAgent完成后返回纯文本，导致主Agent停止")
|
||
print("2. 解决方案: 修改主Agent的系统提示词，明确要求在SubAgent返回后继续执行下一步")
|
||
print("3. 或者: 检查LangGraph的recursion_limit配置，确保允许足够的步骤数")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if len(sys.argv) < 2:
|
||
print("使用方法: python analyze_llm_calls.py <json_file>")
|
||
sys.exit(1)
|
||
|
||
json_file = sys.argv[1]
|
||
analyze_llm_calls(json_file)
|