Index

A

agent_input (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute) agent_inspect module agent_inspect.clients module agent_inspect.clients.llm_client module agent_inspect.exception module agent_inspect.exception.error_codes module agent_inspect.exception.exception module agent_inspect.metrics module agent_inspect.metrics.adapters module agent_inspect.metrics.adapters.base_adapter module agent_inspect.metrics.adapters.tau2bench_adapter module agent_inspect.metrics.adapters.toolsandbox_adapter module agent_inspect.metrics.constants module agent_inspect.metrics.observed module agent_inspect.metrics.observed.latency module agent_inspect.metrics.observed.observed_metric module agent_inspect.metrics.observed.token_count module agent_inspect.metrics.observed.tool_call_count module agent_inspect.metrics.scorer module agent_inspect.metrics.scorer.auc module agent_inspect.metrics.scorer.llm_based_metric module agent_inspect.metrics.scorer.metric module agent_inspect.metrics.scorer.ppt module agent_inspect.metrics.scorer.progress module agent_inspect.metrics.scorer.success module agent_inspect.metrics.scorer.templates module agent_inspect.metrics.scorer.tool_correctness module agent_inspect.metrics.utils module agent_inspect.metrics.utils.expected_tool_validators module agent_inspect.metrics.utils.subgoal_validators module agent_inspect.metrics.utils.trace_validators module agent_inspect.metrics.validator module agent_inspect.metrics.validator.constants module agent_inspect.metrics.validator.exact_match module agent_inspect.metrics.validator.llm_check module agent_inspect.metrics.validator.regex_match module agent_inspect.metrics.validator.subgoal_completion module agent_inspect.metrics.validator.templates module agent_inspect.metrics.validator.tool_call_completion module agent_inspect.metrics.validator.validator module agent_inspect.models module agent_inspect.models.llm_payload module agent_inspect.models.llm_response module agent_inspect.models.metrics module

agent_inspect.models.metrics.agent_data_sample module agent_inspect.models.metrics.agent_trace module agent_inspect.models.metrics.metric_score module agent_inspect.models.metrics.validation_result module agent_inspect.models.tools module agent_inspect.models.tools.analysis_models module agent_inspect.models.tools.error_cluster module agent_inspect.models.user_proxy module agent_inspect.models.user_proxy.chat module agent_inspect.models.user_proxy.terminating_condition module agent_inspect.tools.error_analysis.base_error_analysis module agent_inspect.tools.error_analysis.deterministic_tool_call_error_analysis module agent_inspect.tools.error_analysis.llm_output_schemas module agent_inspect.tools.error_analysis.llm_templates module agent_inspect.tools.error_analysis.semisupervised_subgoal_error_analysis module agent_inspect.tools.error_analysis.semisupervised_tool_call_error_analysis module agent_inspect.tools.error_analysis.subgoal_error_analysis module agent_inspect.tools.error_analysis.tool_call_error_analysis module agent_inspect.tools.error_analysis.unsupervised_subgoal_error_analysis module agent_inspect.user_proxy module agent_inspect.user_proxy.constants module agent_inspect.user_proxy.llm_proxy_agent module agent_inspect.user_proxy.templates module agent_inspect.user_proxy.user_proxy_agent module agent_inspect.user_proxy.utils module agent_inspect.user_proxy.utils.user_proxy_validators module agent_response (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute) agent_responses (agent_inspect.models.user_proxy.chat.ConversationTurn attribute) (agent_inspect.models.user_proxy.ConversationTurn attribute) agent_run_id (agent_inspect.models.tools.analysis_models.AnalyzedValidation attribute) (agent_inspect.models.tools.analysis_models.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.AnalyzedValidation attribute) (agent_inspect.models.tools.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute) agent_thought (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) AgentDialogueTrace (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_trace) AgentResponse (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_trace) analyze_batch() (agent_inspect.tools.error_analysis.subgoal_error_analysis.SubgoalErrorAnalysis method) (agent_inspect.tools.error_analysis.tool_call_error_analysis.ToolCallErrorAnalysis method) analyze_batch_async() (agent_inspect.tools.error_analysis.subgoal_error_analysis.SubgoalErrorAnalysis method) (agent_inspect.tools.error_analysis.tool_call_error_analysis.ToolCallErrorAnalysis method) analyzed_validations_clustered_by_errors (agent_inspect.models.tools.analysis_models.ErrorAnalysisResult attribute) (agent_inspect.models.tools.ErrorAnalysisResult attribute) (agent_inspect.models.tools.SubgoalErrorAnalysisResult attribute) AnalyzedSubgoalValidation (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) AnalyzedToolValidation (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) AnalyzedValidation (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) AUC (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.auc) AUC_CALCULATION_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) AverageLatency (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.latency)

B

base_error (agent_inspect.models.tools.analysis_models.AnalyzedValidation attribute) (agent_inspect.models.tools.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.AnalyzedToolValidation attribute) (agent_inspect.models.tools.AnalyzedValidation attribute)

BaseAdapter (class in agent_inspect.metrics.adapters) (class in agent_inspect.metrics.adapters.base_adapter) BooleanScore (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.metric_score)

C

ChatHistory (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.chat) check (agent_inspect.models.metrics.agent_data_sample.ToolInputParameter attribute) (agent_inspect.models.metrics.agent_data_sample.ToolOutput attribute) (agent_inspect.models.metrics.ToolInputParameter attribute) (agent_inspect.models.metrics.ToolOutput attribute) (agent_inspect.models.user_proxy.chat.UserProxyMessage attribute) (agent_inspect.models.user_proxy.terminating_condition.TaskBlockedTerminatingCondition attribute) (agent_inspect.models.user_proxy.terminating_condition.TaskCompletedTerminatingCondition attribute) (agent_inspect.models.user_proxy.terminating_condition.TaskDelegatedTerminatingCondition attribute) (agent_inspect.models.user_proxy.terminating_condition.TerminatingCondition attribute) (agent_inspect.models.user_proxy.TerminatingCondition attribute) (agent_inspect.models.user_proxy.UserProxyMessage attribute) CLIENT_ERROR_CODE (agent_inspect.exception.ClientComponent attribute) (agent_inspect.exception.error_codes.ClientComponent attribute) CLIENT_REQUEST_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) ClientComponent (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) cluster_label (agent_inspect.models.tools.error_cluster.ErrorCluster attribute) (agent_inspect.models.tools.error_cluster.FaithfulnessErrorCluster attribute) (agent_inspect.models.tools.error_cluster.IncompleteCommunicationCluster attribute) (agent_inspect.models.tools.error_cluster.IncorrectToolInputCluster attribute) (agent_inspect.models.tools.error_cluster.IncorrectToolOutputHandlingCluster attribute) (agent_inspect.models.tools.error_cluster.InstructionFollowingErrorCluster attribute) (agent_inspect.models.tools.error_cluster.LogicalReasoningErrorCluster attribute) (agent_inspect.models.tools.error_cluster.MissedToolCallCluster attribute) (agent_inspect.models.tools.error_cluster.UnclassifiedErrorCluster attribute) (agent_inspect.models.tools.error_cluster.WrongToolSelectionCluster attribute) (agent_inspect.models.tools.ErrorCluster attribute) (agent_inspect.models.tools.FaithfulnessErrorCluster attribute) (agent_inspect.models.tools.IncompleteCommunicationCluster attribute) (agent_inspect.models.tools.IncorrectToolInputCluster attribute) (agent_inspect.models.tools.IncorrectToolOutputHandlingCluster attribute) (agent_inspect.models.tools.InstructionFollowingErrorCluster attribute) (agent_inspect.models.tools.LogicalReasoningErrorCluster attribute) (agent_inspect.models.tools.MissedToolCallCluster attribute) (agent_inspect.models.tools.UnclassifiedErrorCluster attribute) (agent_inspect.models.tools.WrongToolSelectionCluster attribute)

completed_subgoal_validations (agent_inspect.models.tools.analysis_models.SubgoalErrorAnalysisResult attribute) (agent_inspect.models.tools.SubgoalErrorAnalysisResult attribute) completion (agent_inspect.models.llm_response.LLMResponse attribute) (agent_inspect.models.LLMResponse attribute) conversation (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.EvaluationSample attribute) Conversation (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) conversations (agent_inspect.models.user_proxy.chat.ChatHistory attribute) (agent_inspect.models.user_proxy.ChatHistory attribute) ConversationTurn (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.chat) convert_to_agent_trace() (agent_inspect.metrics.adapters.base_adapter.BaseAdapter method) (agent_inspect.metrics.adapters.BaseAdapter method) (agent_inspect.metrics.adapters.tau2bench_adapter.Tau2BenchAdapter method) (agent_inspect.metrics.adapters.Tau2BenchAdapter method) (agent_inspect.metrics.adapters.toolsandbox_adapter.ToolsandboxAdapter method) (agent_inspect.metrics.adapters.ToolsandboxAdapter method)

D

data_sample_id (agent_inspect.models.tools.analysis_models.AnalyzedValidation attribute) (agent_inspect.models.tools.analysis_models.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.AnalyzedToolValidation attribute) (agent_inspect.models.tools.AnalyzedValidation attribute) (agent_inspect.models.tools.ErrorAnalysisDataSample attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute) (agent_inspect.models.tools.SubgoalErrorAnalysisDataSample attribute) (agent_inspect.models.tools.ToolCallErrorAnalysisDataSample attribute) description (agent_inspect.models.tools.error_cluster.ErrorCluster attribute) (agent_inspect.models.tools.error_cluster.FaithfulnessErrorCluster attribute) (agent_inspect.models.tools.error_cluster.IncompleteCommunicationCluster attribute) (agent_inspect.models.tools.error_cluster.IncorrectToolInputCluster attribute) (agent_inspect.models.tools.error_cluster.IncorrectToolOutputHandlingCluster attribute) (agent_inspect.models.tools.error_cluster.InstructionFollowingErrorCluster attribute) (agent_inspect.models.tools.error_cluster.LogicalReasoningErrorCluster attribute) (agent_inspect.models.tools.error_cluster.MissedToolCallCluster attribute) (agent_inspect.models.tools.error_cluster.UnclassifiedErrorCluster attribute) (agent_inspect.models.tools.error_cluster.WrongToolSelectionCluster attribute) (agent_inspect.models.tools.ErrorCluster attribute) (agent_inspect.models.tools.FaithfulnessErrorCluster attribute) (agent_inspect.models.tools.IncompleteCommunicationCluster attribute) (agent_inspect.models.tools.IncorrectToolInputCluster attribute) (agent_inspect.models.tools.IncorrectToolOutputHandlingCluster attribute) (agent_inspect.models.tools.InstructionFollowingErrorCluster attribute) (agent_inspect.models.tools.LogicalReasoningErrorCluster attribute) (agent_inspect.models.tools.MissedToolCallCluster attribute) (agent_inspect.models.tools.UnclassifiedErrorCluster attribute) (agent_inspect.models.tools.WrongToolSelectionCluster attribute)

details (agent_inspect.models.metrics.agent_data_sample.SubGoal attribute) (agent_inspect.models.metrics.SubGoal attribute) DeterministicToolCallErrorAnalysis (class in agent_inspect.tools.error_analysis.deterministic_tool_call_error_analysis)

E

EMPTY_PROGRESS_SCORE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) EMPTY_VALIDATION_RESULT (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) ensure_full_stop() (in module agent_inspect.user_proxy.utils) error_message (agent_inspect.models.llm_response.LLMResponse attribute) (agent_inspect.models.LLMResponse attribute) ErrorAnalysis (class in agent_inspect.tools.error_analysis.base_error_analysis) ErrorAnalysisDataSample (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) ErrorAnalysisResult (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) ErrorCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster) ErrorCode (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) evaluate() (agent_inspect.metrics.observed.AverageLatency method) (agent_inspect.metrics.observed.InputTotalTokenCount method) (agent_inspect.metrics.observed.latency.AverageLatency method) (agent_inspect.metrics.observed.latency.LatencyMetric method) (agent_inspect.metrics.observed.latency.TotalLatency method) (agent_inspect.metrics.observed.observed_metric.ObservedMetric method) (agent_inspect.metrics.observed.ObservedMetric method) (agent_inspect.metrics.observed.OutputTotalTokenCount method) (agent_inspect.metrics.observed.ReasoningTotalTokenCount method) (agent_inspect.metrics.observed.token_count.InputTotalTokenCount method) (agent_inspect.metrics.observed.token_count.OutputTotalTokenCount method) (agent_inspect.metrics.observed.token_count.ReasoningTotalTokenCount method) (agent_inspect.metrics.observed.token_count.TokenConsumptionMetric method) (agent_inspect.metrics.observed.token_count.TotalTokenConsumption method) (agent_inspect.metrics.observed.TokenConsumptionMetric method) (agent_inspect.metrics.observed.tool_call_count.ToolCallCount method) (agent_inspect.metrics.observed.ToolCallCount method) (agent_inspect.metrics.observed.TotalLatency method) (agent_inspect.metrics.observed.TotalTokenConsumption method) (agent_inspect.metrics.scorer.AUC method) (agent_inspect.metrics.scorer.auc.AUC method) (agent_inspect.metrics.scorer.llm_based_metric.LLMBasedMetric method) (agent_inspect.metrics.scorer.LLMBasedMetric method) (agent_inspect.metrics.scorer.Metric method) (agent_inspect.metrics.scorer.metric.Metric method) (agent_inspect.metrics.scorer.PPT method) (agent_inspect.metrics.scorer.ppt.PPT method) (agent_inspect.metrics.scorer.progress.ProgressBasedMetric method) (agent_inspect.metrics.scorer.progress.ProgressScore method) (agent_inspect.metrics.scorer.progress.ProgressScoresThroughTurns method) (agent_inspect.metrics.scorer.ProgressBasedMetric method) (agent_inspect.metrics.scorer.ProgressScore method) (agent_inspect.metrics.scorer.ProgressScoresThroughTurns method) (agent_inspect.metrics.scorer.success.SuccessBasedMetric method) (agent_inspect.metrics.scorer.success.SuccessScore method) (agent_inspect.metrics.scorer.success.SuccessScoreFinalTurn method) (agent_inspect.metrics.scorer.SuccessBasedMetric method) (agent_inspect.metrics.scorer.SuccessScore method) (agent_inspect.metrics.scorer.SuccessScoreFinalTurn method) (agent_inspect.metrics.scorer.tool_correctness.ToolCorrectnessMetric method) (agent_inspect.metrics.scorer.ToolCorrectnessMetric method)

EVALUATION_ERROR_CODE (agent_inspect.exception.error_codes.EvaluationComponent attribute) (agent_inspect.exception.EvaluationComponent attribute) EvaluationComponent (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) EvaluationError, [1] EvaluationSample (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) exact_match() (in module agent_inspect.metrics.validator) (in module agent_inspect.metrics.validator.exact_match) expected_output (agent_inspect.models.metrics.agent_data_sample.ExpectedToolCall attribute) (agent_inspect.models.metrics.ExpectedToolCall attribute) expected_parameters (agent_inspect.models.metrics.agent_data_sample.ExpectedToolCall attribute) (agent_inspect.models.metrics.ExpectedToolCall attribute) expected_response (agent_inspect.models.metrics.agent_data_sample.Conversation attribute) (agent_inspect.models.metrics.Conversation attribute) expected_tool_call (agent_inspect.models.metrics.ToolCallValidationResult attribute) (agent_inspect.models.metrics.validation_result.ToolCallValidationResult attribute) expected_tool_calls (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.EvaluationSample attribute) ExpectedToolCall (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) ExpectedToolCallValidator (class in agent_inspect.metrics.utils.expected_tool_validators) explanations (agent_inspect.models.metrics.BooleanScore attribute) (agent_inspect.models.metrics.metric_score.BooleanScore attribute) (agent_inspect.models.metrics.metric_score.NumericalScore attribute) (agent_inspect.models.metrics.NumericalScore attribute) (agent_inspect.models.metrics.SubGoalValidationResult attribute) (agent_inspect.models.metrics.ToolCallValidationResult attribute) (agent_inspect.models.metrics.validation_result.ValidationResult attribute) (agent_inspect.models.metrics.ValidationResult attribute)

F

FaithfulnessErrorCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster)

from_id (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute)

G

generate_message_from_chat_history() (agent_inspect.user_proxy.llm_proxy_agent.LLMProxyAgent method) (agent_inspect.user_proxy.user_proxy_agent.UserProxyAgent method) (agent_inspect.user_proxy.UserProxyAgent method) get_auc_score_from_progress_scores() (agent_inspect.metrics.scorer.AUC static method) (agent_inspect.metrics.scorer.auc.AUC static method) get_majority_voted_score_from_judge_responses() (agent_inspect.metrics.validator.Validator method) (agent_inspect.metrics.validator.validator.Validator method)

get_ppt_score_from_progress_scores() (agent_inspect.metrics.scorer.PPT static method) (agent_inspect.metrics.scorer.ppt.PPT static method) get_success_score_from_progress_score() (agent_inspect.metrics.scorer.success.SuccessBasedMetric static method) (agent_inspect.metrics.scorer.SuccessBasedMetric static method) get_success_score_from_validation_results() (agent_inspect.metrics.scorer.success.SuccessBasedMetric static method) (agent_inspect.metrics.scorer.SuccessBasedMetric static method)

I

id (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.EvaluationSample attribute) (agent_inspect.models.metrics.Step attribute) (agent_inspect.models.metrics.TurnTrace attribute) (agent_inspect.models.user_proxy.chat.ChatHistory attribute) (agent_inspect.models.user_proxy.chat.ConversationTurn attribute) (agent_inspect.models.user_proxy.ChatHistory attribute) (agent_inspect.models.user_proxy.ConversationTurn attribute) IncompleteCommunicationCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster) IncorrectToolInputCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster) IncorrectToolOutputHandlingCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster) input_token_consumption (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) InputTotalTokenCount (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count) InstructionFollowingErrorCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster)

INSUFFICIENT_JUDGE_RESPONSES_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_ERROR_ANALYSIS_ERROR_CLUSTERS (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_JSON_DECODE_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_JUDGE_RESPONSE_FORMAT_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_JUDGE_TRIALS_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_LLM_JUDGE_RESULT_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_USER_MESSAGE_REFLECTION (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_USER_PROXY_RESPONSE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) INVALID_VALUE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) InvalidInputValueError, [1] is_completed (agent_inspect.models.metrics.SubGoalValidationResult attribute) (agent_inspect.models.metrics.ToolCallValidationResult attribute) (agent_inspect.models.metrics.validation_result.ValidationResult attribute) (agent_inspect.models.metrics.ValidationResult attribute)

J

json_data (agent_inspect.models.user_proxy.chat.ResponseFromAgent attribute) (agent_inspect.models.user_proxy.ResponseFromAgent attribute) judge_expectation (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute)

judge_std (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute)

L

latency_in_ms (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute) LatencyMetric (class in agent_inspect.metrics.observed.latency) llm_check() (in module agent_inspect.metrics.validator) (in module agent_inspect.metrics.validator.llm_check) LLMBasedMetric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.llm_based_metric) LLMClient (class in agent_inspect.clients) (class in agent_inspect.clients.llm_client)

LLMPayload (class in agent_inspect.models) (class in agent_inspect.models.llm_payload) LLMProxyAgent (class in agent_inspect.user_proxy.llm_proxy_agent) LLMResponse (class in agent_inspect.models) (class in agent_inspect.models.llm_response) LogicalReasoningErrorCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster)

M

make_llm_request() (agent_inspect.clients.llm_client.LLMClient method) (agent_inspect.clients.LLMClient method) make_llm_requests() (agent_inspect.clients.llm_client.LLMClient method) (agent_inspect.clients.LLMClient method) make_request_with_payload() (agent_inspect.clients.llm_client.LLMClient method) (agent_inspect.clients.LLMClient method) MAX_RETRY_EXCEEDED_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) max_tokens (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) message (agent_inspect.models.metrics.agent_data_sample.Conversation attribute) (agent_inspect.models.metrics.Conversation attribute) message_str (agent_inspect.models.user_proxy.chat.UserProxyMessage attribute) (agent_inspect.models.user_proxy.UserProxyMessage attribute) Metric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.metric) MissedToolCallCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster) MISSING_VALUE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) model (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) module agent_inspect agent_inspect.clients agent_inspect.clients.llm_client agent_inspect.exception agent_inspect.exception.error_codes agent_inspect.exception.exception agent_inspect.metrics agent_inspect.metrics.adapters agent_inspect.metrics.adapters.base_adapter agent_inspect.metrics.adapters.tau2bench_adapter agent_inspect.metrics.adapters.toolsandbox_adapter agent_inspect.metrics.constants agent_inspect.metrics.observed agent_inspect.metrics.observed.latency agent_inspect.metrics.observed.observed_metric agent_inspect.metrics.observed.token_count agent_inspect.metrics.observed.tool_call_count agent_inspect.metrics.scorer agent_inspect.metrics.scorer.auc agent_inspect.metrics.scorer.llm_based_metric agent_inspect.metrics.scorer.metric agent_inspect.metrics.scorer.ppt agent_inspect.metrics.scorer.progress agent_inspect.metrics.scorer.success agent_inspect.metrics.scorer.templates agent_inspect.metrics.scorer.tool_correctness agent_inspect.metrics.utils agent_inspect.metrics.utils.expected_tool_validators agent_inspect.metrics.utils.subgoal_validators agent_inspect.metrics.utils.trace_validators agent_inspect.metrics.validator agent_inspect.metrics.validator.constants agent_inspect.metrics.validator.exact_match agent_inspect.metrics.validator.llm_check agent_inspect.metrics.validator.regex_match agent_inspect.metrics.validator.subgoal_completion agent_inspect.metrics.validator.templates agent_inspect.metrics.validator.tool_call_completion agent_inspect.metrics.validator.validator agent_inspect.models agent_inspect.models.llm_payload agent_inspect.models.llm_response agent_inspect.models.metrics agent_inspect.models.metrics.agent_data_sample agent_inspect.models.metrics.agent_trace agent_inspect.models.metrics.metric_score agent_inspect.models.metrics.validation_result agent_inspect.models.tools agent_inspect.models.tools.analysis_models agent_inspect.models.tools.error_cluster agent_inspect.models.user_proxy agent_inspect.models.user_proxy.chat agent_inspect.models.user_proxy.terminating_condition agent_inspect.tools.error_analysis.base_error_analysis agent_inspect.tools.error_analysis.deterministic_tool_call_error_analysis agent_inspect.tools.error_analysis.llm_output_schemas agent_inspect.tools.error_analysis.llm_templates agent_inspect.tools.error_analysis.semisupervised_subgoal_error_analysis agent_inspect.tools.error_analysis.semisupervised_tool_call_error_analysis agent_inspect.tools.error_analysis.subgoal_error_analysis agent_inspect.tools.error_analysis.tool_call_error_analysis agent_inspect.tools.error_analysis.unsupervised_subgoal_error_analysis agent_inspect.user_proxy agent_inspect.user_proxy.constants agent_inspect.user_proxy.llm_proxy_agent agent_inspect.user_proxy.templates agent_inspect.user_proxy.user_proxy_agent agent_inspect.user_proxy.utils agent_inspect.user_proxy.utils.user_proxy_validators

N

name (agent_inspect.models.metrics.agent_data_sample.ToolInputParameter attribute) (agent_inspect.models.metrics.ToolInputParameter attribute)

NumericalScore (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.metric_score)

O

ObservedMetric (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.observed_metric) output_token_consumption (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute)

OutputTotalTokenCount (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count)

P

parent_ids (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) PPT (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.ppt) PPT_CALCULATION_ERROR (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) ProgressBasedMetric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.progress)

ProgressScore (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.progress) ProgressScoresThroughTurns (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.progress) prompt_sent_to_llmj (agent_inspect.models.metrics.SubGoalValidationResult attribute) (agent_inspect.models.metrics.validation_result.SubGoalValidationResult attribute)

R

reasoning_token_consumption (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) ReasoningTotalTokenCount (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count) regex_match() (in module agent_inspect.metrics.validator) (in module agent_inspect.metrics.validator.regex_match)

response (agent_inspect.models.metrics.agent_trace.AgentResponse attribute) (agent_inspect.models.metrics.AgentResponse attribute) response_str (agent_inspect.models.user_proxy.chat.ResponseFromAgent attribute) (agent_inspect.models.user_proxy.ResponseFromAgent attribute) ResponseFromAgent (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.chat)

S

score (agent_inspect.models.metrics.BooleanScore attribute) (agent_inspect.models.metrics.metric_score.BooleanScore attribute) (agent_inspect.models.metrics.metric_score.NumericalScore attribute) (agent_inspect.models.metrics.NumericalScore attribute) SemisupervisedSubgoalErrorAnalysis (class in agent_inspect.tools.error_analysis.semisupervised_subgoal_error_analysis) SemisupervisedToolCallErrorAnalysis (class in agent_inspect.tools.error_analysis.semisupervised_tool_call_error_analysis) StatisticAnalysisResult (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) status (agent_inspect.models.llm_response.LLMResponse attribute) (agent_inspect.models.LLMResponse attribute) status_code (agent_inspect.models.metrics.agent_trace.AgentResponse attribute) (agent_inspect.models.metrics.AgentResponse attribute) Step (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_trace) steps (agent_inspect.models.metrics.agent_trace.TurnTrace attribute) (agent_inspect.models.metrics.TurnTrace attribute) stop_sequence (agent_inspect.models.user_proxy.terminating_condition.TaskBlockedTerminatingCondition attribute) (agent_inspect.models.user_proxy.terminating_condition.TaskCompletedTerminatingCondition attribute) (agent_inspect.models.user_proxy.terminating_condition.TaskDelegatedTerminatingCondition attribute) (agent_inspect.models.user_proxy.terminating_condition.TerminatingCondition attribute) (agent_inspect.models.user_proxy.TerminatingCondition attribute) structured_output (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) sub_goal (agent_inspect.models.metrics.SubGoalValidationResult attribute) (agent_inspect.models.metrics.validation_result.SubGoalValidationResult attribute) sub_goals (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.EvaluationSample attribute)

sub_scores (agent_inspect.models.metrics.metric_score.NumericalScore attribute) (agent_inspect.models.metrics.NumericalScore attribute) SubGoal (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) subgoal_validation (agent_inspect.models.tools.analysis_models.AnalyzedSubgoalValidation attribute) (agent_inspect.models.tools.AnalyzedSubgoalValidation attribute) subgoal_validations (agent_inspect.models.tools.analysis_models.StatisticAnalysisResult attribute) (agent_inspect.models.tools.analysis_models.SubgoalErrorAnalysisDataSample attribute) (agent_inspect.models.tools.StatisticAnalysisResult attribute) (agent_inspect.models.tools.SubgoalErrorAnalysisDataSample attribute) SubGoalCompletionValidator (class in agent_inspect.metrics.validator) (class in agent_inspect.metrics.validator.subgoal_completion) SubgoalErrorAnalysis (class in agent_inspect.tools.error_analysis.subgoal_error_analysis) SubgoalErrorAnalysisDataSample (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) SubgoalErrorAnalysisResult (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) SubGoalValidationResult (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.validation_result) SubGoalValidator (class in agent_inspect.metrics.utils.subgoal_validators) SuccessBasedMetric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.success) SuccessScore (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.success) SuccessScoreFinalTurn (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.success) system_prompt (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute)

T

TaskBlockedTerminatingCondition (class in agent_inspect.models.user_proxy.terminating_condition) TaskCompletedTerminatingCondition (class in agent_inspect.models.user_proxy.terminating_condition) TaskDelegatedTerminatingCondition (class in agent_inspect.models.user_proxy.terminating_condition) Tau2BenchAdapter (class in agent_inspect.metrics.adapters) (class in agent_inspect.metrics.adapters.tau2bench_adapter) temperature (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) TerminatingCondition (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.terminating_condition) to_dict() (agent_inspect.models.tools.error_cluster.ErrorCluster method) (agent_inspect.models.tools.ErrorCluster method) TokenConsumptionMetric (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count) tool (agent_inspect.models.metrics.agent_data_sample.ExpectedToolCall attribute) (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.ExpectedToolCall attribute) (agent_inspect.models.metrics.Step attribute) tool_call_validation (agent_inspect.models.tools.analysis_models.AnalyzedToolValidation attribute) (agent_inspect.models.tools.AnalyzedToolValidation attribute) tool_call_validations (agent_inspect.models.tools.analysis_models.ToolCallErrorAnalysisDataSample attribute) (agent_inspect.models.tools.ToolCallErrorAnalysisDataSample attribute) TOOL_ERROR_CODE (agent_inspect.exception.error_codes.ToolComponent attribute) (agent_inspect.exception.ToolComponent attribute) tool_input_args (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) tool_output (agent_inspect.models.metrics.agent_trace.Step attribute) (agent_inspect.models.metrics.Step attribute) ToolCallCompletionValidator (class in agent_inspect.metrics.validator) (class in agent_inspect.metrics.validator.tool_call_completion) ToolCallCount (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.tool_call_count) ToolCallErrorAnalysis (class in agent_inspect.tools.error_analysis.tool_call_error_analysis)

ToolCallErrorAnalysisDataSample (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) ToolCallErrorAnalysisResult (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.analysis_models) ToolCallValidationResult (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.validation_result) ToolComponent (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) ToolCorrectnessMetric (class in agent_inspect.metrics.scorer) (class in agent_inspect.metrics.scorer.tool_correctness) ToolError, [1] ToolInputParameter (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) ToolOutput (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_data_sample) ToolsandboxAdapter (class in agent_inspect.metrics.adapters) (class in agent_inspect.metrics.adapters.toolsandbox_adapter) TotalLatency (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.latency) TotalTokenConsumption (class in agent_inspect.metrics.observed) (class in agent_inspect.metrics.observed.token_count) TraceValidator (class in agent_inspect.metrics.utils.trace_validators) turn (agent_inspect.models.metrics.agent_data_sample.ExpectedToolCall attribute) (agent_inspect.models.metrics.agent_data_sample.SubGoal attribute) (agent_inspect.models.metrics.ExpectedToolCall attribute) (agent_inspect.models.metrics.SubGoal attribute) turn_id (agent_inspect.models.metrics.agent_data_sample.Conversation attribute) (agent_inspect.models.metrics.Conversation attribute) turns (agent_inspect.models.metrics.agent_trace.AgentDialogueTrace attribute) (agent_inspect.models.metrics.AgentDialogueTrace attribute) TurnTrace (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.agent_trace)

U

UnclassifiedErrorCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster) UNSUCCESSFUL_LLM_SUMMARIZATION (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) UNSUCCESSFUL_MAJORITY_VOTING (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) UnsupervisedSubgoalErrorAnalysis (class in agent_inspect.tools.error_analysis.unsupervised_subgoal_error_analysis) UNSUPPORTED_ATTRIBUTION_TYPE (agent_inspect.exception.error_codes.ErrorCode attribute) (agent_inspect.exception.ErrorCode attribute) user_instruction (agent_inspect.models.metrics.agent_data_sample.EvaluationSample attribute) (agent_inspect.models.metrics.EvaluationSample attribute) user_message (agent_inspect.models.user_proxy.chat.ConversationTurn attribute) (agent_inspect.models.user_proxy.ConversationTurn attribute)

user_prompt (agent_inspect.models.llm_payload.LLMPayload attribute) (agent_inspect.models.LLMPayload attribute) USER_PROXY_ERROR_CODE (agent_inspect.exception.error_codes.UserProxyComponent attribute) (agent_inspect.exception.UserProxyComponent attribute) UserProxyAgent (class in agent_inspect.user_proxy) (class in agent_inspect.user_proxy.user_proxy_agent) UserProxyComponent (class in agent_inspect.exception) (class in agent_inspect.exception.error_codes) UserProxyError, [1] UserProxyInputValidator (class in agent_inspect.user_proxy.utils.user_proxy_validators) UserProxyMessage (class in agent_inspect.models.user_proxy) (class in agent_inspect.models.user_proxy.chat)

V

validate() (agent_inspect.metrics.validator.subgoal_completion.SubGoalCompletionValidator method) (agent_inspect.metrics.validator.SubGoalCompletionValidator method) (agent_inspect.metrics.validator.Validator method) (agent_inspect.metrics.validator.validator.Validator method) validate_dynamic() (agent_inspect.metrics.validator.subgoal_completion.SubGoalCompletionValidator method) (agent_inspect.metrics.validator.SubGoalCompletionValidator method) validate_multi_turns() (agent_inspect.metrics.validator.tool_call_completion.ToolCallCompletionValidator method) (agent_inspect.metrics.validator.ToolCallCompletionValidator method) validate_static_last_turn() (agent_inspect.metrics.validator.tool_call_completion.ToolCallCompletionValidator method) (agent_inspect.metrics.validator.ToolCallCompletionValidator method)

validation_results (agent_inspect.models.metrics.metric_score.NumericalScore attribute) (agent_inspect.models.metrics.NumericalScore attribute) ValidationResult (class in agent_inspect.models.metrics) (class in agent_inspect.models.metrics.validation_result) Validator (class in agent_inspect.metrics.validator) (class in agent_inspect.metrics.validator.validator) value (agent_inspect.models.metrics.agent_data_sample.ToolInputParameter attribute) (agent_inspect.models.metrics.agent_data_sample.ToolOutput attribute) (agent_inspect.models.metrics.ToolInputParameter attribute) (agent_inspect.models.metrics.ToolOutput attribute)

W

WrongToolSelectionCluster (class in agent_inspect.models.tools) (class in agent_inspect.models.tools.error_cluster)