如何监控 AI 智能体：CloudWatch 基础设施、Arize Phoenix 追踪与 LLM-as-Judge 质量评估

import phoenix as px
from opentelemetry import trace as trace_api
from opentelemetry.sdk import trace as trace_sdk
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from openinference.instrumentation.bedrock import BedrockInstrumentor

# 启动本地 Phoenix 应用
session = px.launch_app()

# 配置 OTel 追踪提供者
tracer_provider = trace_sdk.TracerProvider()
tracer_provider.add_span_processor(
    SimpleSpanProcessor(OTLPSpanExporter(endpoint="http://localhost:6006/v1/traces"))
)
trace_api.set_tracer_provider(tracer_provider)

# 自动插桩：捕获所有 Bedrock/LLM 调用
BedrockInstrumentor().instrument(tracer_provider=tracer_provider)

import boto3
import time

class AgentMonitor:
    def __init__(self, namespace="AI/Agents"):
        self.cw = boto3.client("cloudwatch", region_name="us-east-1")
        self.namespace = namespace

    def record_metrics(self, agent_name, latency, tokens, is_success):
        metrics = [
            {"MetricName": "Latency", "Value": latency, "Unit": "Milliseconds"},
            {"MetricName": "Tokens", "Value": tokens, "Unit": "Count"},
            {"MetricName": "Success", "Value": 1 if is_success else 0, "Unit": "Count"}
        ]
        self.cw.put_metric_data(
            Namespace=self.namespace,
            MetricData=metrics,
            Dimensions=[{"Name": "AgentName", "Value": agent_name}]
        )

from phoenix.evals import LLM, create_evaluator, evaluate_dataframe
import pandas as pd

# 配置评委模型，建议使用 n1n.ai 接入的高性能模型
eval_model = LLM(provider="bedrock", model="us.anthropic.claude-3-5-sonnet-v1")

@create_evaluator(name="帮助性评分", source="llm")
def helpfulness_eval(input: str, output: str) -> float:
    prompt = (
        f"请作为一名公正的裁判，评价以下 AI 回答的帮助性。\n"
        f"用户问题: {input}\n"
        f"AI 回答: {output}\n"
        f"请给出 0.0 到 1.0 之间的分数，仅返回数字。"
    )
    result = eval_model.generate_text(prompt=prompt)
    try:
        return float(result.strip())
    except:
        return 0.5