Generative AI Custom Evaluation

This page showcases how to use the Evaluations SDK to benchmark LLMs, evaluate orchestration configurations, or prompts. It uses the publicly available MedicationQA dataset, which consists of commonly asked consumer questions about medications. The workload computes industry-standard metrics to check the reliability of the responses generated by the LLM.

Environment Setup

Add the following additional variables into your .env file to provide the AWS credentials and S3 bucket information for the evaluation job:

AICORE_AWS_ACCESS_KEY_ID=<your-aws-access-key-id>
AICORE_AWS_SECRET_ACCESS_KEY=<your-aws-secret-access-key>
aws_access_key_id=<your-aws-access-key-id>
aws_secret_access_key=<your-aws-secret-access-key>
AWS_BUCKET_ID=<your-s3-bucket-id>
AWS_REGION=<your-aws-region>

Setup

from gen_ai_hub.evaluations import EvaluationClient
from dotenv import load_dotenv
import os

load_dotenv(override=True)
client = EvaluationClient.from_env()
# Please ensure that the following environment variables are set in your .env file.
# client = EvaluationClient(
#    base_url=os.getenv("AICORE_BASE_URL"),
#    auth_url=os.getenv("AICORE_AUTH_URL"),
#    client_id=os.getenv("AICORE_CLIENT_ID"),
#    client_secret=os.getenv("AICORE_CLIENT_SECRET"),
#    resource_group=os.getenv("AICORE_RESOURCE_GROUP"),
#    aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
#    aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
#    orchestration_url=os.getenv("ORCHESTRATION_URL")
#)

# print(client.base_url)
# print(client.ai_core_client.object_store_secrets.query(
#    top=10, resource_group=os.getenv("AICORE_RESOURCE_GROUP")
# ))

Note: If you are using the default resource group, an orchestration URL is already available and will be selected automatically. If you have previously created object store secrets, the setup step below can be skipped.

One-Time Setup: Create Secrets

import json

AWS_S3_ENDPOINT = "s3-eu-central-1.amazonaws.com"

default_secret_creds = {
    "data": {}, "type": "S3", "pathPrefix": "sdkOutputFiles",
    "endpoint": AWS_S3_ENDPOINT, "bucket": os.getenv("AWS_BUCKET_ID"),
    "region": os.getenv("AWS_REGION"), "usehttps": "1",
}
input_secret_creds = {
    "data": {}, "name": "sdk-data", "type": "S3", "pathPrefix": "sdk_input_files/data",
    "endpoint": AWS_S3_ENDPOINT, "bucket": os.getenv("AWS_BUCKET_ID"),
    "region": os.getenv("AWS_REGION"), "usehttps": "1",
}

response = client.setup(
    default_secret_body=default_secret_creds,
    input_secret_body=input_secret_creds,
    replace_existing=True
)
print(json.dumps(response, indent=4, ensure_ascii=False))

List Available Models and Metrics

models_list = client.list_available_models()
print([m["model"] for m in models_list])

metrics_list = client.get_system_supported_metrics()
print([m["name"] for m in metrics_list])

Define Evaluation Config

To know more about the structure and type of the parameters in any of the functions, you can call help() on it and see the docstring. For example, for EvaluationConfig:

from gen_ai_hub.evaluations import EvaluationConfig
print(help(EvaluationConfig))

There are multiple ways to create an EvaluationConfig — with a prompt template (inline or reference) or an orchestration registry reference, and with multiple metric configurations in each EvaluationConfig.

from gen_ai_hub.prompt_registry import PromptTemplateSpec, PromptTemplate
from gen_ai_hub.orchestration_v2.models.template_ref import TemplateRef, TemplateRefByID
from gen_ai_hub.evaluations import Dataset, MetricConfig, MetricRef, EvaluationConfig
from gen_ai_hub.orchestration_v2.models.llm_model_details import LLMModelDetails

evaluation_config_list = [
    EvaluationConfig(
        llm=LLMModelDetails(name="gpt-5", version="latest"),
        template=PromptTemplateSpec(
            template=[PromptTemplate(
                role="user",
                content="Provide a concise and informative response to the following consumer health question: {{?question}}"
            )]
        ),
        template_variable_mapping={"question": "topic"},
        dataset_config=Dataset("eval-data/testdata/medicalqna_dataset.csv"),
        metrics=[
            MetricConfig(
                reference=MetricRef(id="36d1abca-cf01-48f6-9bd1-8d5e1272a374"),  # Pointwise Correctness
                variable_mapping={"reference": "ground_truth"},
            ),
        ],
    ),
    EvaluationConfig(
        # The name of an orchestration configuration registered in the Orchestration Registry.
        orchestration_registry_reference="<ORCHESTRATION REGISTRY REFERENCE>",
        template_variable_mapping={"question": "topic"},
        dataset_config=Dataset("eval-data/testdata/medicalqna_dataset.csv"),
        metrics=[
            MetricConfig(
                reference=MetricRef(id="36d1abca-cf01-48f6-9bd1-8d5e1272a374"),  # Pointwise Correctness
                variable_mapping={"reference": "ground_truth"},
            ),
            MetricConfig(reference=MetricRef(id="ba2ece64-d4ed-4645-96d3-8728af12515f")),  # Content Filter on Input
        ],
    ),
    EvaluationConfig(
        llm=LLMModelDetails(name="gpt-5", version="latest"),
        template=TemplateRef(template_ref=TemplateRefByID(id="<TEMPLATE REF ID>")),
        template_variable_mapping={"question": "topic"},
        dataset_config=Dataset("eval-data/testdata/medicalqna_dataset.csv"),
        metrics=[
            MetricConfig(reference=MetricRef(id="cd3ffd21-faae-4f06-8184-52541182d9a5")),  # Pointwise Instruction Following
        ],
    ),
]

Run Evaluation

evaluation_runs = client.evaluate(evaluation_config_list)

for current_run in evaluation_runs:
    print(f"id: {current_run.id}, status: {current_run.status}")

Wait for Completion

for current_run in evaluation_runs:
    print("Waiting for run:", current_run.id)
    current_run.wait_for_completion(timeout=3600)

Debugging

To debug the evaluation job in case of failures, you can use these helper methods on the run object to get more details.

import json

debug_info = current_run.get_debug_info()
print("Debug info:", debug_info)

debug_logs = current_run.get_debug_logs()
print("Logs:", json.dumps(debug_logs, indent=4, default=str))

View Results

Aggregate Results

run1_data = evaluation_runs[0].results()
run2_data = evaluation_runs[1].results().aggregations()
run3_data = evaluation_runs[2].results().aggregations()
print(run1_data)

Completion Responses

data = evaluation_runs[0].results().completions()

def wrap_column(df, col, col_px=200):
    df = df.drop(columns=["created_at", "updated_at"])
    return (
        df.style.set_table_styles([
            {"selector": "table", "props": "table-layout: fixed; width: 100%;"},
            {
                "selector": f"td.col{df.columns.get_loc(col)}",
                "props": f"max-width: {col_px}px; white-space: pre-wrap; word-break: break-word;",
            },
        ], overwrite=False)
    )

wrap_column(data.head(), "completion_result", col_px=700)

Metric Evaluation

data = evaluation_runs[0].results().metrics()

def wrap_column(df, col, col_px=200):
    df = df.drop(columns=["created_at", "updated_at"])
    return (
        df.style.set_table_styles([
            {"selector": "table", "props": "table-layout: fixed; width: 100%;"},
            {
                "selector": f"td.col{df.columns.get_loc(col)}",
                "props": f"max-width: {col_px}px; white-space: pre-wrap; word-break: break-word;",
            },
        ], overwrite=False)
    )

wrap_column(data.head(), "metric_result", col_px=700)