建立自動調度資源叢集

建立具有自動調度資源政策的 Dataproc 叢集。

程式碼範例

Java

在試用這個範例之前，請先按照「使用用戶端程式庫的 Managed Service for Apache Spark 快速入門導覽課程」中的 Java 設定說明操作。詳情請參閱 Managed Service for Apache Spark Java API 參考文件。

如要向 Managed Service for Apache Spark 進行驗證，請設定應用程式預設憑證。詳情請參閱「為本機開發環境設定驗證機制」。

import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.dataproc.v1.AutoscalingConfig;
import com.google.cloud.dataproc.v1.AutoscalingPolicy;
import com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClient;
import com.google.cloud.dataproc.v1.AutoscalingPolicyServiceSettings;
import com.google.cloud.dataproc.v1.BasicAutoscalingAlgorithm;
import com.google.cloud.dataproc.v1.BasicYarnAutoscalingConfig;
import com.google.cloud.dataproc.v1.Cluster;
import com.google.cloud.dataproc.v1.ClusterConfig;
import com.google.cloud.dataproc.v1.ClusterControllerClient;
import com.google.cloud.dataproc.v1.ClusterControllerSettings;
import com.google.cloud.dataproc.v1.ClusterOperationMetadata;
import com.google.cloud.dataproc.v1.InstanceGroupAutoscalingPolicyConfig;
import com.google.cloud.dataproc.v1.InstanceGroupConfig;
import com.google.cloud.dataproc.v1.RegionName;
import com.google.protobuf.Duration;
import java.io.IOException;
import java.util.concurrent.ExecutionException;

public class CreateClusterWithAutoscaling {

  public static void createClusterwithAutoscaling() throws IOException, InterruptedException {
    // TODO(developer): Replace these variables before running the sample.
    String projectId = "your-project-id";
    String region = "your-project-region";
    String clusterName = "your-cluster-name";
    String autoscalingPolicyName = "your-autoscaling-policy";
    createClusterwithAutoscaling(projectId, region, clusterName, autoscalingPolicyName);
  }

  public static void createClusterwithAutoscaling(
      String projectId, String region, String clusterName, String autoscalingPolicyName)
      throws IOException, InterruptedException {
    String myEndpoint = String.format("%s-dataproc.googleapis.com:443", region);

    // Configure the settings for the cluster controller client.
    ClusterControllerSettings clusterControllerSettings =
        ClusterControllerSettings.newBuilder().setEndpoint(myEndpoint).build();

    // Configure the settings for the autoscaling policy service client.
    AutoscalingPolicyServiceSettings autoscalingPolicyServiceSettings =
        AutoscalingPolicyServiceSettings.newBuilder().setEndpoint(myEndpoint).build();

    // Create a cluster controller client and an autoscaling controller client with the configured
    // settings. The clients only need to be created once and can be reused for multiple requests.
    // Using a
    // try-with-resources closes the client, but this can also be done manually with the .close()
    // method.
    try (ClusterControllerClient clusterControllerClient =
            ClusterControllerClient.create(clusterControllerSettings);
        AutoscalingPolicyServiceClient autoscalingPolicyServiceClient =
            AutoscalingPolicyServiceClient.create(autoscalingPolicyServiceSettings)) {

      // Create the Autoscaling policy.
      InstanceGroupAutoscalingPolicyConfig workerInstanceGroupAutoscalingPolicyConfig =
          InstanceGroupAutoscalingPolicyConfig.newBuilder()
              .setMinInstances(2)
              .setMaxInstances(100)
              .setWeight(1)
              .build();
      InstanceGroupAutoscalingPolicyConfig secondaryWorkerInstanceGroupAutoscalingPolicyConfig =
          InstanceGroupAutoscalingPolicyConfig.newBuilder()
              .setMinInstances(0)
              .setMaxInstances(100)
              .setWeight(1)
              .build();
      BasicYarnAutoscalingConfig basicYarnApplicationConfig =
          BasicYarnAutoscalingConfig.newBuilder()
              .setScaleUpFactor(0.05)
              .setScaleDownFactor(1.0)
              .setScaleUpMinWorkerFraction(0.0)
              .setScaleUpMinWorkerFraction(0.0)
              .setGracefulDecommissionTimeout(Duration.newBuilder().setSeconds(3600).build())
              .build();
      BasicAutoscalingAlgorithm basicAutoscalingAlgorithm =
          BasicAutoscalingAlgorithm.newBuilder()
              .setCooldownPeriod(Duration.newBuilder().setSeconds(240).build())
              .setYarnConfig(basicYarnApplicationConfig)
              .build();
      AutoscalingPolicy autoscalingPolicy =
          AutoscalingPolicy.newBuilder()
              .setId(autoscalingPolicyName)
              .setWorkerConfig(workerInstanceGroupAutoscalingPolicyConfig)
              .setSecondaryWorkerConfig(secondaryWorkerInstanceGroupAutoscalingPolicyConfig)
              .setBasicAlgorithm(basicAutoscalingAlgorithm)
              .build();
      RegionName parent = RegionName.of(projectId, region);

      // Policy is uploaded here.
      autoscalingPolicyServiceClient.createAutoscalingPolicy(parent, autoscalingPolicy);

      // Now the policy can be referenced when creating a cluster.
      String autoscalingPolicyUri =
          String.format(
              "projects/%s/locations/%s/autoscalingPolicies/%s",
              projectId, region, autoscalingPolicyName);
      AutoscalingConfig autoscalingConfig =
          AutoscalingConfig.newBuilder().setPolicyUri(autoscalingPolicyUri).build();

      // Configure the settings for our cluster.
      InstanceGroupConfig masterConfig =
          InstanceGroupConfig.newBuilder()
              .setMachineTypeUri("n1-standard-2")
              .setNumInstances(1)
              .build();
      InstanceGroupConfig workerConfig =
          InstanceGroupConfig.newBuilder()
              .setMachineTypeUri("n1-standard-2")
              .setNumInstances(2)
              .build();
      ClusterConfig clusterConfig =
          ClusterConfig.newBuilder()
              .setMasterConfig(masterConfig)
              .setWorkerConfig(workerConfig)
              .setAutoscalingConfig(autoscalingConfig)
              .build();

      // Create the cluster object with the desired cluster config.
      Cluster cluster =
          Cluster.newBuilder().setClusterName(clusterName).setConfig(clusterConfig).build();

      // Create the Dataproc cluster.
      OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest =
          clusterControllerClient.createClusterAsync(projectId, region, cluster);
      Cluster response = createClusterAsyncRequest.get();

      // Print out a success message.
      System.out.printf("Cluster created successfully: %s", response.getClusterName());

    } catch (ExecutionException e) {
      // If cluster creation does not complete successfully, print the error message.
      System.err.println(String.format("createClusterWithAutoscaling: %s ", e.getMessage()));
    }
  }
}

後續步驟

如要搜尋及篩選其他 Google Cloud 產品的程式碼範例，請參閱Google Cloud 範例瀏覽工具。