This code sample demonstrates how to extract video embeddings from a specific segment of a video using the MultiModal Embedding model.

## Explore further


For detailed documentation that includes this code sample, see the following:

- [Get multimodal embeddings](https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/embeddings/get-multimodal-embeddings)
- [Multimodal embeddings API](https://docs.cloud.google.com/gemini-enterprise-agent-platform/reference/models/multimodal-embeddings-api)

## Code sample

### Go


Before trying this sample, follow the Go setup instructions in the
[Vertex AI quickstart using
client libraries](https://docs.cloud.google.com/vertex-ai/docs/start/client-libraries).


For more information, see the
[Vertex AI Go API
reference documentation](https://docs.cloud.google.com/go/docs/reference/cloud.google.com/go/aiplatform/latest/apiv1).


To authenticate to Vertex AI, set up Application Default Credentials.
For more information, see

[Set up authentication for a local development environment](https://docs.cloud.google.com/docs/authentication/set-up-adc-local-dev-environment).

    import (
    	"context"
    	"encoding/json"
    	"fmt"
    	"io"
    	"time"

    	aiplatform "cloud.google.com/go/aiplatform/apiv1beta1"
    	aiplatformpb "cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb"
    	"google.golang.org/api/option"
    	"google.golang.org/protobuf/encoding/protojson"
    	"google.golang.org/protobuf/types/known/structpb"
    )

    // generateForVideo shows how to use the multimodal model to generate embeddings for video input.
    func generateForVideo(w io.Writer, project, location string) error {
    	// location = "us-central1"

    	// The default context timeout may be not enough to process a video input.
    	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
    	defer cancel()

    	apiEndpoint := fmt.Sprintf("%s-aiplatform.googleapis.com:443", location)
    	client, err := aiplatform.https://docs.cloud.google.com/go/docs/reference/cloud.google.com/go/aiplatform/latest/apiv1beta1.html#cloud_google_com_go_aiplatform_apiv1beta1_PredictionClient_NewPredictionClient(ctx, option.WithEndpoint(apiEndpoint))
    	if err != nil {
    		return fmt.Errorf("failed to construct API client: %w", err)
    	}
    	defer client.Close()

    	model := "multimodalembedding@001"
    	endpoint := fmt.Sprintf("projects/%s/locations/%s/publishers/google/models/%s", project, location, model)

    	// This is the input to the model's prediction call. For schema, see:
    	// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#request_body
    	instances, err := structpb.NewValue(map[string]any{
    		"video": map[string]any{
    			// Video input can be provided either as a Google Cloud Storage URI or as base64-encoded
    			// bytes using the "bytesBase64Encoded" field.
    			"gcsUri": "gs://cloud-samples-data/vertex-ai-vision/highway_vehicles.mp4",
    			"videoSegmentConfig": map[string]any{
    				"startOffsetSec": 1,
    				"endOffsetSec":   5,
    			},
    		},
    	})
    	if err != nil {
    		return fmt.Errorf("failed to construct request payload: %w", err)
    	}

    	req := &aiplatformpb.PredictRequest{
    		Endpoint: endpoint,
    		// The model supports only 1 instance per request.
    		Instances: []*structpb.Value{instances},
    	}
    	resp, err := client.Predict(ctx, req)
    	if err != nil {
    		return fmt.Errorf("failed to generate embeddings: %w", err)
    	}

    	instanceEmbeddingsJson, err := protojson.Marshal(resp.GetPredictions()[0])
    	if err != nil {
    		return fmt.Errorf("failed to convert protobuf value to JSON: %w", err)
    	}
    	// For response schema, see:
    	// https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-embeddings-api#response-body
    	var instanceEmbeddings struct {
    		VideoEmbeddings []struct {
    			Embedding      []float32 `json:"embedding"`
    			StartOffsetSec float64   `json:"startOffsetSec"`
    			EndOffsetSec   float64   `json:"endOffsetSec"`
    		} `json:"videoEmbeddings"`
    	}
    	if err := json.Unmarshal(instanceEmbeddingsJson, &instanceEmbeddings); err != nil {
    		return fmt.Errorf("failed to unmarshal json: %w", err)
    	}
    	// Get the embedding for our single video segment (`.videoEmbeddings` object has one entry per
    	// each processed segment).
    	videoEmbedding := instanceEmbeddings.VideoEmbeddings[0]

    	fmt.Fprintf(w, "Video embedding (seconds: %.f-%.f; length=%d): %v\n",
    		videoEmbedding.StartOffsetSec,
    		videoEmbedding.EndOffsetSec,
    		len(videoEmbedding.Embedding),
    		videoEmbedding.Embedding,
    	)
    	// Example response:
    	// Video embedding (seconds: 1-5; length=1408): [-0.016427778 0.032878537 -0.030755188 ... ]

    	return nil
    }

### Python


Before trying this sample, follow the Python setup instructions in the
[Vertex AI quickstart using
client libraries](https://docs.cloud.google.com/vertex-ai/docs/start/client-libraries).


For more information, see the
[Vertex AI Python API
reference documentation](https://docs.cloud.google.com/python/docs/reference/aiplatform/latest).


To authenticate to Vertex AI, set up Application Default Credentials.
For more information, see

[Set up authentication for a local development environment](https://docs.cloud.google.com/docs/authentication/set-up-adc-local-dev-environment).

    import https://docs.cloud.google.com/python/docs/reference/vertexai/latest

    from vertexai.vision_models import https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.MultiModalEmbeddingModel.html, https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.Video.html
    from vertexai.vision_models import https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.VideoSegmentConfig.html

    # TODO(developer): Update & uncomment line below
    # PROJECT_ID = "your-project-id"
    https://docs.cloud.google.com/python/docs/reference/vertexai/latest.init(project=PROJECT_ID, location="us-central1")

    model = https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.MultiModalEmbeddingModel.html.https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.MultiModalEmbeddingModel.html#vertexai_preview_vision_models_MultiModalEmbeddingModel_from_pretrained("multimodalembedding@001")

    embeddings = model.https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.MultiModalEmbeddingModel.html#vertexai_preview_vision_models_MultiModalEmbeddingModel_get_embeddings(
        video=https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.Video.html.https://docs.cloud.google.com/python/docs/reference/vertexai/latest/vertexai.preview.vision_models.Video.html#vertexai_preview_vision_models_Video_load_from_file(
            "gs://cloud-samples-data/vertex-ai-vision/highway_vehicles.mp4"
        ),
        video_segment_config=VideoSegmentConfig(end_offset_sec=1),
    )

    # Video Embeddings are segmented based on the video_segment_config.
    print("Video Embeddings:")
    for video_embedding in embeddings.video_embeddings:
        print(
            f"Video Segment: {video_embedding.start_offset_sec} - {video_embedding.end_offset_sec}"
        )
        print(f"Embedding: {video_embedding.embedding}")

    # Example response:
    # Video Embeddings:
    # Video Segment: 0.0 - 1.0
    # Embedding: [-0.0206376351, 0.0123456789, ...]

## What's next


To search and filter code samples for other Google Cloud products, see the
[Google Cloud sample browser](https://docs.cloud.google.com/docs/samples?product=generativeaionvertexai).