Evaluate a model

Evaluate the performance of a generative AI model.

Explore further

For detailed documentation that includes this code sample, see the following:

Code sample

Python

Before trying this sample, follow the Python setup instructions in the Vertex AI quickstart using client libraries. For more information, see the Vertex AI Python API reference documentation.

To authenticate to Vertex AI, set up Application Default Credentials. For more information, see Set up authentication for a local development environment. 

import os

import vertexai
from vertexai.evaluation import EvalResult, EvalTask

# TODO (Developer) Set environment variables
PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
LOCATION_ID = os.getenv("LOCATION_ID", "us-central1")
MODEL_NAME = os.getenv("MODEL_NAME", "gemini-2.5-flash")


def evaluate_model() -> EvalResult:
    """Evaluate the performance of a generative AI model."""

    vertexai.init(project=PROJECT_ID, location=LOCATION_ID)

    # Dataset URI containing input prompts and ground truth labels
    dataset_uri = "gs://cloud-samples-data/ai-platform/generative_ai/llm_classification_bp_input_prompts_with_ground_truth.jsonl"

    metric_column_mapping = {"reference": "ground_truth"}

    # Define evaluation task
    eval_task = EvalTask(
        dataset=dataset_uri,
        metrics=["exact_match"],
        experiment="gemini-classification-eval",
        metric_column_mapping=metric_column_mapping,
    )

    # Define a prompt template so the generative Gemini model produces formatted labels
    prompt_template = (
        "Classify the following text into exactly one category from "
        "[nature, news, sports, health, startups]. Return only the category name:\n\n{prompt}"
    )

    # Evaluate using a modern Gemini model
    eval_result = eval_task.evaluate(
        model=MODEL_NAME,
        prompt_template=prompt_template,
    )

    print("=== SUMMARY METRICS ===")
    print(eval_result.summary_metrics)

    print("\n=== METRICS TABLE SAMPLE ===")
    print(eval_result.metrics_table.head())

    return eval_result

What's next

To search and filter code samples for other Google Cloud products, see the Google Cloud sample browser.