This code sample shows how to use the multimodal model to generate embeddings for text and image inputs.

## Explore further


For detailed documentation that includes this code sample, see the following:

- [Get multimodal embeddings](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/embeddings/get-multimodal-embeddings)
- [Multimodal embeddings API](https://docs.cloud.google.com/gemini-enterprise-agent-platform/reference/models/multimodal-embeddings-api)

## Code sample

### Go


Before trying this sample, follow the Go setup instructions in the
[Vertex AI quickstart using
client libraries](https://docs.cloud.google.com/vertex-ai/docs/start/client-libraries).


For more information, see the
[Vertex AI Go API
reference documentation](https://docs.cloud.google.com/go/docs/reference/cloud.google.com/go/aiplatform/latest/apiv1).


To authenticate to Vertex AI, set up Application Default Credentials.
For more information, see

[Set up authentication for a local development environment](https://docs.cloud.google.com/docs/authentication/set-up-adc-local-dev-environment).

    import (
    	"context"
    	"encoding/json"
    	"fmt"
    	"io"

    	aiplatform "cloud.google.com/go/aiplatform/apiv1beta1"
    	aiplatformpb "cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb"
    	"google.golang.org/api/option"
    	"google.golang.org/protobuf/encoding/protojson"
    	"google.golang.org/protobuf/types/known/structpb"
    )

    // generateForTextAndImage shows how to use the multimodal model to generate embeddings for
    // text and image inputs.
    func generateForTextAndImage(w io.Writer, project, location string) error {
    	// location = "us-central1"
    	ctx := context.Background()
    	apiEndpoint := fmt.Sprintf("%s-aiplatform.googleapis.com:443", location)
    	client, err := aiplatform.https://docs.cloud.google.com/go/docs/reference/cloud.google.com/go/aiplatform/latest/apiv1beta1.html#cloud_google_com_go_aiplatform_apiv1beta1_PredictionClient_NewPredictionClient(ctx, option.WithEndpoint(apiEndpoint))
    	if err != nil {
    		return fmt.Errorf("failed to construct API client: %w", err)
    	}
    	defer client.Close()

    	model := "multimodalembedding@001"
    	endpoint := fmt.Sprintf("projects/%s/locations/%s/publishers/google/models/%s", project, location, model)

    	// This is the input to the model's prediction call. For schema, see:
    	// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#request_body
    	instance, err := structpb.NewValue(map[string]any{
    		"image": map[string]any{
    			// Image input can be provided either as a Google Cloud Storage URI or as
    			// base64-encoded bytes using the "bytesBase64Encoded" field.
    			"gcsUri": "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png",
    		},
    		"text": "Colosseum",
    	})
    	if err != nil {
    		return fmt.Errorf("failed to construct request payload: %w", err)
    	}

    	req := &aiplatformpb.PredictRequest{
    		Endpoint: endpoint,
    		// The model supports only 1 instance per request.
    		Instances: []*structpb.Value{instance},
    	}

    	resp, err := client.Predict(ctx, req)
    	if err != nil {
    		return fmt.Errorf("failed to generate embeddings: %w", err)
    	}

    	instanceEmbeddingsJson, err := protojson.Marshal(resp.GetPredictions()[0])
    	if err != nil {
    		return fmt.Errorf("failed to convert protobuf value to JSON: %w", err)
    	}
    	// For response schema, see:
    	// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#response-body
    	var instanceEmbeddings struct {
    		ImageEmbeddings []float32 `json:"imageEmbedding"`
    		TextEmbeddings  []float32 `json:"textEmbedding"`
    	}
    	if err := json.Unmarshal(instanceEmbeddingsJson, &instanceEmbeddings); err != nil {
    		return fmt.Errorf("failed to unmarshal JSON: %w", err)
    	}

    	imageEmbedding := instanceEmbeddings.ImageEmbeddings
    	textEmbedding := instanceEmbeddings.TextEmbeddings

    	fmt.Fprintf(w, "Text embedding (length=%d): %v\n", len(textEmbedding), textEmbedding)
    	fmt.Fprintf(w, "Image embedding (length=%d): %v\n", len(imageEmbedding), imageEmbedding)
    	// Example response:
    	// Text embedding (length=1408): [0.0023026613 0.027898183 -0.011858357 ... ]
    	// Image embedding (length=1408): [-0.012314269 0.07271844 0.00020170923 ... ]

    	return nil
    }

### Python


Before trying this sample, follow the Python setup instructions in the
[Vertex AI quickstart using
client libraries](https://docs.cloud.google.com/vertex-ai/docs/start/client-libraries).


For more information, see the
[Vertex AI Python API
reference documentation](https://docs.cloud.google.com/python/docs/reference/aiplatform/latest).


To authenticate to Vertex AI, set up Application Default Credentials.
For more information, see

[Set up authentication for a local development environment](https://docs.cloud.google.com/docs/authentication/set-up-adc-local-dev-environment).

    import https://docs.cloud.google.com/python/docs/reference/vertexai/latest
    from vertexai.vision_models import https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.generative_models.Image.html, https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.MultiModalEmbeddingModel.html

    # TODO(developer): Update & uncomment line below
    # PROJECT_ID = "your-project-id"
    https://docs.cloud.google.com/python/docs/reference/vertexai/latest.init(project=PROJECT_ID, location="us-central1")

    model = https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.MultiModalEmbeddingModel.html.https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.MultiModalEmbeddingModel.html#vertexai_preview_vision_models_MultiModalEmbeddingModel_from_pretrained("multimodalembedding@001")
    image = https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.generative_models.Image.html.https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.generative_models.Image.html#vertexai_preview_generative_models_Image_load_from_file(
        "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"
    )

    embeddings = model.https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.MultiModalEmbeddingModel.html#vertexai_preview_vision_models_MultiModalEmbeddingModel_get_embeddings(
        image=image,
        contextual_text="Colosseum",
        dimension=1408,
    )
    print(f"Image Embedding: {embeddings.image_embedding}")
    print(f"Text Embedding: {embeddings.text_embedding}")
    # Example response:
    # Image Embedding: [-0.0123147098, 0.0727171078, ...]
    # Text Embedding: [0.00230263756, 0.0278981831, ...]

## What's next


To search and filter code samples for other Google Cloud products, see the
[Google Cloud sample browser](https://docs.cloud.google.com/docs/samples?product=generativeaionvertexai).