gen_ai_hub.evaluations.utils package

Submodules

• gen_ai_hub.evaluations.utils.aicore_utils module
  • generate_random_id()
  • find_configuration_id_by_name()
  • get_all_configurations()
  • get_running_deployments_by_configuration_id()
  • create_deployment_by_configuration_id()
  • create_llm_orchestration_deployment_url()
  • wait_for_target_status()
  • read_data_from_artifact()
  • build_s3_file_key()
  • resolve_artifact_path()
  • fetch_deployment_config()
  • fetch_configuration_by_id()
  • call_orchestration_service_with_v2_config()
  • upload_file_to_aws_s3()
  • upload_evaluation_dataset_data()
  • register_aicore_artifact()
  • register_aicore_configuration()
  • register_aicore_execution()
  • list_available_llm_models()
  • fetch_orchestration_config_from_registry()
  • resolve_metric_identifiers()
  • resolve_metric_names()
• gen_ai_hub.evaluations.utils.config_data_utils module
  • get_orch_config_data()
  • get_dataset_data()
• gen_ai_hub.evaluations.utils.file_utils module
  • load_config_file()
  • read_local_csv_file()
• gen_ai_hub.evaluations.utils.gen_utils module
  • update_variable_mapping()
  • get_accumulated_config_data()
  • set_model_details_from_run_configs()
  • create_model_versions_map_from_orch_configs()
  • parse_model_filter_list()
  • build_model_versions_map()
  • create_model_versions_map_from_configuration_param_bindings()
  • create_model_versions_map_from_custom_metric_config()
  • select_model_details_randomly()
  • update_test_orch_config()
  • has_filter_key()
  • get_filter_config()
  • check_if_content_filter_provider_supported()
  • remove_filter_metrics_if_provider_not_supported()
  • create_custom_metric_name()
  • validate_metric_name()
  • count_user_prompts_from_template_list()
  • get_template_list_from_orch_config()
  • validate_prompts_in_templating_module()
  • get_custom_metric_ids_from_input()
  • check_if_metric_is_defined()
  • is_value_in_json()
  • validate_metrics()
  • list_prompt_variables()
  • extract_dataset_columns()
  • get_prompt_variables_from_orch_config()
  • get_grounding_config_from_orch_config()
  • get_grounding_output_param_key()
  • get_defaults()
  • get_mapped_value_if_exists()
  • validate_variable_mapping_of_prompts()
  • validate_all_metrics_mapping()
  • extract_metrics_variables()
  • validate_individual_metrics()
  • validate_variable_mapping_of_metrics()
  • flatten_prompt_configuration()
  • validate_individual_custom_metrics()
  • handle_missing_dependent_variables_in_dataset()
  • populate_dataset_data_if_data_missing()
  • populate_dataset_data_if_single_schema_provided()
  • handle_json_schema_match()
  • validate_language_code_and_data_population()
  • handle_language_match()
  • populate_dataset_data_if_single_reference_provided()
  • populate_dataset_data_if_individual_metric_reference_provided()
  • handle_reference_missing_rows()
  • update_artifact_dict()
  • resolve_orchestration_config_v2()
• gen_ai_hub.evaluations.utils.language_match_utils module
  • LanguageMapper
    • LanguageMapper.EXCLUDED_CODES
    • LanguageMapper.get_iso_code_639_1()
• gen_ai_hub.evaluations.utils.metric_client_utils module
  • get_custom_metric_by_id()
  • get_metric_template_info_from_server()
  • get_metric_version_history()
  • fetch_all_system_predefined_metrics()
• gen_ai_hub.evaluations.utils.orch_config_utils module
  • validate_mandatory_modules()
  • validate_orch_config_mandatory_modules()
  • get_model_name()
  • validate_model_name()
  • validate_model_name_in_llm_module_config()
  • get_prompt_templating_config()
  • validate_template_ref_absent_in_config()
  • get_template_key()
  • validate_if_template_list_is_empty()
  • validate_if_template_list_is_empty_in_templating_module_config()
  • get_template_list_from_orch_config()
  • validate_if_content_inside_template_is_empty_in_templating_module_config()
  • validate_if_image_url_is_provided_in_content_type_inside_templating_module_config()
  • validate_if_grounding_output_present_in_prompt_variables()
  • validate_if_all_grounding_input_params_present_in_prompt_variables()
  • validate_orchestration_params_from_evaluation_config()
  • to_comparable()
• gen_ai_hub.evaluations.utils.oss_secret_utils module
  • ObjectStoreData
    • ObjectStoreData.provider_name
    • ObjectStoreData.aws_access_key_id
    • ObjectStoreData.aws_secret_access_key
    • ObjectStoreData.__init__()
  • create_aws_object_store_secret()
  • fetch_object_store_secret_by_name()
  • delete_object_store_secret()
• gen_ai_hub.evaluations.utils.validation_utils module
  • validate_filtered_models()
  • fetch_and_validate_orchestration_config()
  • validate_orchestration_url_across_configs()
  • extract_deployment_id()
  • validate_orchestration_url()
  • validate_orchestration_configuration()
  • validate_input_config()
  • validate_variable_mapping_with_input_config()
  • validate_config_data_collection()
  • validate_merged_config_data()