""" 记录LLM调用的详细信息 - 保存为JSON文件 使用方法： export PYTHONIOENCODING=utf-8 && python tests/debug_llm_calls.py """ import sys import os import json from datetime import datetime from typing import Any, Dict, List from uuid import UUID # 添加项目根目录到Python路径 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from langchain_core.callbacks import BaseCallbackHandler from langchain_core.messages import BaseMessage from langchain_core.outputs import LLMResult from src.agents.coordinator import create_research_coordinator from src.config import Config class LLMCallLogger(BaseCallbackHandler): """记录所有LLM调用的回调处理器""" def __init__(self): self.calls: List[Dict[str, Any]] = [] self.current_call = None self.call_count = 0 def on_llm_start( self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any ) -> None: """LLM开始时调用""" self.call_count += 1 self.current_call = { "call_id": self.call_count, "timestamp_start": datetime.now().isoformat(), "prompts": prompts, "kwargs": {k: str(v) for k, v in kwargs.items() if k != "invocation_params"}, } print(f"\n{'='*80}") print(f"🔵 LLM调用 #{self.call_count} 开始 - {datetime.now().strftime('%H:%M:%S')}") print('='*80) if prompts: print(f"Prompt长度: {len(prompts[0])} 字符") print(f"Prompt预览: {prompts[0][:200]}...") def on_chat_model_start( self, serialized: Dict[str, Any], messages: List[List[BaseMessage]], **kwargs: Any ) -> None: """Chat模型开始时调用""" self.call_count += 1 self.current_call = { "call_id": self.call_count, "timestamp_start": datetime.now().isoformat(), "messages": [ [ { "type": type(msg).__name__, "content": msg.content if hasattr(msg, 'content') else str(msg), "tool_calls": getattr(msg, 'tool_calls', None) } for msg in msg_list ] for msg_list in messages ], "kwargs": {k: str(v) for k, v in kwargs.items() if k not in ["invocation_params", "tags", "metadata"]}, } print(f"\n{'='*80}") print(f"🔵 Chat模型调用 #{self.call_count} 开始 - {datetime.now().strftime('%H:%M:%S')}") print('='*80) if messages: print(f"消息数量: {len(messages[0])}") for i, msg in enumerate(messages[0][-3:], 1): msg_type = type(msg).__name__ print(f" {i}. {msg_type}: {str(msg.content)[:100] if hasattr(msg, 'content') else 'N/A'}...") def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None: """LLM结束时调用""" if self.current_call: self.current_call["timestamp_end"] = datetime.now().isoformat() # 提取响应 generations = [] for gen_list in response.generations: for gen in gen_list: gen_info = { "text": gen.text if hasattr(gen, 'text') else None, } if hasattr(gen, 'message'): msg = gen.message gen_info["message"] = { "type": type(msg).__name__, "content": msg.content if hasattr(msg, 'content') else None, "tool_calls": [ { "name": tc.get("name"), "args": tc.get("args"), "id": tc.get("id") } for tc in (msg.tool_calls if hasattr(msg, 'tool_calls') and msg.tool_calls else []) ] if hasattr(msg, 'tool_calls') else None } generations.append(gen_info) self.current_call["response"] = { "generations": generations, "llm_output": response.llm_output, } self.calls.append(self.current_call) print(f"\n✅ LLM调用 #{self.current_call['call_id']} 完成") if generations: gen = generations[0] if gen.get("message"): msg = gen["message"] print(f"响应类型: {msg['type']}") if msg.get('content'): print(f"内容: {msg['content'][:150]}...") if msg.get('tool_calls'): print(f"工具调用: {len(msg['tool_calls'])} 个") for tc in msg['tool_calls'][:3]: print(f" - {tc['name']}") self.current_call = None def on_llm_error(self, error: Exception, **kwargs: Any) -> None: """LLM出错时调用""" if self.current_call: self.current_call["timestamp_end"] = datetime.now().isoformat() self.current_call["error"] = str(error) self.calls.append(self.current_call) print(f"\n❌ LLM调用 #{self.current_call['call_id']} 出错: {error}") self.current_call = None def save_to_file(self, filepath: str): """保存记录到JSON文件""" with open(filepath, 'w', encoding='utf-8') as f: json.dump({ "total_calls": len(self.calls), "calls": self.calls }, f, ensure_ascii=False, indent=2) print(f"\n💾 已保存 {len(self.calls)} 次LLM调用记录到: {filepath}") def test_with_llm_logging(question: str, depth: str = "quick", max_steps: int = 10): """ 测试研究流程，记录所有LLM调用 Args: question: 研究问题 depth: 深度模式 max_steps: 最大执行步骤数（防止无限循环） """ print("\n" + "🔬 " * 40) print("智能深度研究系统 - LLM调用记录模式") print("🔬 " * 40) print(f"\n研究问题: {question}") print(f"深度模式: {depth}") print(f"最大步骤数: {max_steps}") print(f"开始时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") # 创建日志记录器 logger = LLMCallLogger() # 创建Agent（带callback） print("\n" + "="*80) print("创建Agent...") print("="*80) try: # 获取LLM并添加callback llm = Config.get_llm() llm.callbacks = [logger] # 创建Agent agent = create_research_coordinator( question=question, depth=depth, format="technical", min_tier=3 ) print("✅ Agent创建成功") except Exception as e: print(f"❌ Agent创建失败: {e}") import traceback traceback.print_exc() return # 执行研究 print("\n" + "="*80) print(f"执行研究流程（最多{max_steps}步）...") print("="*80) try: start_time = datetime.now() step_count = 0 # 使用stream模式，但限制步骤数 for chunk in agent.stream( { "messages": [ { "role": "user", "content": f"请开始研究这个问题：{question}" } ] }, config={"callbacks": [logger]} ): step_count += 1 print(f"\n{'─'*80}") print(f"📍 步骤 #{step_count} - {datetime.now().strftime('%H:%M:%S')}") print('─'*80) # 显示state更新 if isinstance(chunk, dict): if 'messages' in chunk: print(f" 消息: {len(chunk['messages'])} 条") if 'files' in chunk: print(f" 文件: {len(chunk['files'])} 个") for path in list(chunk['files'].keys())[:3]: print(f" - {path}") # 限制步骤数 if step_count >= max_steps: print(f"\n⚠️ 达到最大步骤数 {max_steps}，停止执行") break # 超时保护 elapsed = (datetime.now() - start_time).total_seconds() if elapsed > 120: # 2分钟 print(f"\n⚠️ 超过2分钟，停止执行") break end_time = datetime.now() duration = (end_time - start_time).total_seconds() print("\n" + "="*80) print("执行结束") print("="*80) print(f"总步骤数: {step_count}") print(f"LLM调用次数: {len(logger.calls)}") print(f"总耗时: {duration:.2f}秒") except KeyboardInterrupt: print("\n\n⚠️ 用户中断") except Exception as e: print(f"\n\n❌ 执行失败: {e}") import traceback traceback.print_exc() finally: # 保存日志 timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") output_dir = "tests" os.makedirs(output_dir, exist_ok=True) log_file = os.path.join(output_dir, f"llm_calls_{timestamp}.json") logger.save_to_file(log_file) # 也保存一份摘要 summary_file = os.path.join(output_dir, f"llm_calls_summary_{timestamp}.txt") with open(summary_file, 'w', encoding='utf-8') as f: f.write(f"LLM调用记录摘要\n") f.write(f"{'='*80}\n\n") f.write(f"总调用次数: {len(logger.calls)}\n") f.write(f"执行时长: {duration:.2f}秒\n\n") for i, call in enumerate(logger.calls, 1): f.write(f"\n{'─'*80}\n") f.write(f"调用 #{i}\n") f.write(f"{'─'*80}\n") f.write(f"开始: {call['timestamp_start']}\n") f.write(f"结束: {call.get('timestamp_end', 'N/A')}\n") if 'messages' in call: f.write(f"消息数: {len(call['messages'][0]) if call['messages'] else 0}\n") if 'response' in call: gens = call['response'].get('generations', []) if gens: gen = gens[0] if gen.get('message'): msg = gen['message'] f.write(f"响应类型: {msg['type']}\n") if msg.get('tool_calls'): f.write(f"工具调用: {[tc['name'] for tc in msg['tool_calls']]}\n") if 'error' in call: f.write(f"错误: {call['error']}\n") print(f"📄 摘要已保存到: {summary_file}") if __name__ == "__main__": question = "Python asyncio最佳实践" # 只执行前几步，不做完整research test_with_llm_logging(question, depth="quick", max_steps=10)