diff options
Diffstat (limited to 'third_party/googleapis/google/cloud/dataproc')
14 files changed, 0 insertions, 5441 deletions
diff --git a/third_party/googleapis/google/cloud/dataproc/BUILD.bazel b/third_party/googleapis/google/cloud/dataproc/BUILD.bazel deleted file mode 100644 index 192ecee..0000000 --- a/third_party/googleapis/google/cloud/dataproc/BUILD.bazel +++ /dev/null @@ -1,41 +0,0 @@ -# This build file includes a target for the Ruby wrapper library for -# google-cloud-dataproc. - -# This is an API workspace, having public visibility by default makes perfect sense. -package(default_visibility = ["//visibility:public"]) - -# Export yaml configs. -exports_files(glob(["*.yaml"])) - -load( - "@com_google_googleapis_imports//:imports.bzl", - "ruby_cloud_gapic_library", - "ruby_gapic_assembly_pkg", -) - -# Generates a Ruby wrapper client for dataproc. -# Ruby wrapper clients are versionless, but are generated from source protos -# for a particular service version, v1 in this case. -ruby_cloud_gapic_library( - name = "dataproc_ruby_wrapper", - srcs = ["//google/cloud/dataproc/v1:dataproc_proto_with_info"], - extra_protoc_parameters = [ - "ruby-cloud-gem-name=google-cloud-dataproc", - "ruby-cloud-env-prefix=DATAPROC", - "ruby-cloud-wrapper-of=v1:0.7", - "ruby-cloud-product-url=https://cloud.google.com/dataproc", - "ruby-cloud-api-id=dataproc.googleapis.com", - "ruby-cloud-api-shortname=dataproc", - "ruby-cloud-migration-version=1.0", - ], - ruby_cloud_description = "Manages Hadoop-based clusters and jobs on Google Cloud Platform.", - ruby_cloud_title = "Cloud Dataproc", -) - -# Open Source package. -ruby_gapic_assembly_pkg( - name = "google-cloud-dataproc-ruby", - deps = [ - ":dataproc_ruby_wrapper", - ], -) diff --git a/third_party/googleapis/google/cloud/dataproc/logging/BUILD.bazel b/third_party/googleapis/google/cloud/dataproc/logging/BUILD.bazel deleted file mode 100644 index 88780e4..0000000 --- a/third_party/googleapis/google/cloud/dataproc/logging/BUILD.bazel +++ /dev/null @@ -1,176 +0,0 @@ -# This file was automatically generated by BuildFileGenerator - -# This is an API workspace, having public visibility by default makes perfect sense. -package(default_visibility = ["//visibility:public"]) - -############################################################################## -# Common -############################################################################## -load("@rules_proto//proto:defs.bzl", "proto_library") - -proto_library( - name = "logging_proto", - srcs = [ - "autoscaler_log.proto", - ], - deps = [ - "@com_google_protobuf//:duration_proto", - ], -) - -############################################################################## -# Java -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "java_grpc_library", - "java_proto_library", -) - -java_proto_library( - name = "logging_java_proto", - deps = [":logging_proto"], -) - -java_grpc_library( - name = "logging_java_grpc", - srcs = [":logging_proto"], - deps = [":logging_java_proto"], -) - -############################################################################## -# Go -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "go_proto_library", -) - -go_proto_library( - name = "logging_go_proto", - compilers = ["@io_bazel_rules_go//proto:go_grpc"], - importpath = "google.golang.org/genproto/googleapis/cloud/dataproc/logging", - protos = [":logging_proto"], - deps = [ - ], -) - -############################################################################## -# Python -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "moved_proto_library", - "py_grpc_library", - "py_proto_library", -) - -moved_proto_library( - name = "logging_moved_proto", - srcs = [":logging_proto"], - deps = [ - "@com_google_protobuf//:duration_proto", - ], -) - -py_proto_library( - name = "logging_py_proto", - deps = [":logging_moved_proto"], -) - -py_grpc_library( - name = "logging_py_grpc", - srcs = [":logging_moved_proto"], - deps = [":logging_py_proto"], -) - -############################################################################## -# PHP -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "php_grpc_library", - "php_proto_library", -) - -php_proto_library( - name = "logging_php_proto", - deps = [":logging_proto"], -) - -php_grpc_library( - name = "logging_php_grpc", - srcs = [":logging_proto"], - deps = [":logging_php_proto"], -) - -############################################################################## -# Node.js -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "nodejs_gapic_assembly_pkg", - "nodejs_gapic_library", -) - - -############################################################################## -# Ruby -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "ruby_grpc_library", - "ruby_proto_library", -) - -ruby_proto_library( - name = "logging_ruby_proto", - deps = [":logging_proto"], -) - -ruby_grpc_library( - name = "logging_ruby_grpc", - srcs = [":logging_proto"], - deps = [":logging_ruby_proto"], -) - -############################################################################## -# C# -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "csharp_grpc_library", - "csharp_proto_library", -) - -csharp_proto_library( - name = "logging_csharp_proto", - deps = [":logging_proto"], -) - -csharp_grpc_library( - name = "logging_csharp_grpc", - srcs = [":logging_proto"], - deps = [":logging_csharp_proto"], -) - -############################################################################## -# C++ -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "cc_grpc_library", - "cc_proto_library", -) - -cc_proto_library( - name = "logging_cc_proto", - deps = [":logging_proto"], -) - -cc_grpc_library( - name = "logging_cc_grpc", - srcs = [":logging_proto"], - grpc_only = True, - deps = [":logging_cc_proto"], -) diff --git a/third_party/googleapis/google/cloud/dataproc/logging/autoscaler_log.proto b/third_party/googleapis/google/cloud/dataproc/logging/autoscaler_log.proto deleted file mode 100644 index eb3dc0e..0000000 --- a/third_party/googleapis/google/cloud/dataproc/logging/autoscaler_log.proto +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.logging; - -import "google/protobuf/duration.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/logging;logging"; -option java_multiple_files = true; -option java_package = "com.google.cloud.dataproc.logging"; - -// The short version of cluster configuration for Cloud logging. -message ClusterSize { - // The number of primary workers in the cluster. - int32 primary_worker_count = 1; - - // The number of secondary workers in the cluster. - int32 secondary_worker_count = 2; -} - -// The main proto that will be converted to JSON format and then written to -// Logging. -message AutoscalerLog { - // The current Autoscaler status. - AutoscalerStatus status = 1; - - // Optional. The autoscaling recommendation including its inputs, outputs, - // scaling decision, and detailed explanation. - AutoscalerRecommendation recommendation = 2; -} - -// The Autoscaler state. -enum AutoscalerState { - AUTOSCALER_STATE_UNSPECIFIED = 0; - - // The Autoscaler is sleeping and waiting for the next update. - COOLDOWN = 1; - - // The Autoscaler is in the process of calculating its recommendation on - // whether to scale the cluster, and if so, how to autoscale. - RECOMMENDING = 6; - - // The Autoscaler is scaling the cluster. - SCALING = 2; - - // The Autoscaler has stopped. - STOPPED = 3; - - // The Autoscaler has failed. - FAILED = 4; - - // The Autoscaler is initializing. - INITIALIZING = 5; -} - -// The Autoscaling decision type. -enum ScalingDecisionType { - SCALING_DECISION_TYPE_UNSPECIFIED = 0; - - // Increase the number of primary and/or secondary workers. - SCALE_UP = 1; - - // Decrease the number of primary and/or secondary workers. - SCALE_DOWN = 2; - - // Not changing the number of primary or secondary workers. - NO_SCALE = 3; - - // Scale the primary and secondary worker groups in different directions. - MIXED = 4; -} - -enum ConstrainingFactor { - CONSTRAINING_FACTOR_UNSPECIFIED = 0; - - // The project does not have sufficient regional, global, and or preemptible - // quota to allocate a new VM. - SCALING_CAPPED_DUE_TO_LACK_OF_QUOTA = 1; - - // All worker groups have reached maximum size. This message will not be - // issued if one group reached maximum size, but workers were able to be - // allocated to another group. - REACHED_MAXIMUM_CLUSTER_SIZE = 2; - - // All worker groups have reached minimum size. This message will not be - // issued if workers were able to be removed from another group that had not - // reached minimum size. - REACHED_MINIMUM_CLUSTER_SIZE = 3; -} - -// The Autoscaler's status, including its state and other details. -message AutoscalerStatus { - // The high-level Autoscaler state. - AutoscalerState state = 1; - - // The detailed description of Autoscaler status. - string details = 2; - - // The cluster update operation ID. - string update_cluster_operation_id = 3; - - // Error message from an Autoscaler exception, if any. - string error = 4; -} - -// The inputs, outputs, and detailed explanation of the Autoscaling -// recommendation. -message AutoscalerRecommendation { - // The input values for the Autoscaling recommendation alogirthm. - message Inputs { - // The metrics collected by the Dataproc agent running on the cluster. - // For example, {"avg-yarn-pending-memory": "1040 MB"} - map<string, string> cluster_metrics = 1; - - // The cluster configuration before updating the cluster. - ClusterSize current_cluster_size = 2; - - // The minimum worker counts for each instance group. - ClusterSize min_worker_counts = 3; - - // The maximum worker counts for each instance group. - ClusterSize max_worker_counts = 4; - } - - // Autoscaler recommendations. - message Outputs { - // The high-level autoscaling decision, such as SCALE_UP, SCALE_DOWN, - // NO_OP. - ScalingDecisionType decision = 1; - - // The recommended cluster size. - ClusterSize recommended_cluster_size = 2; - - // The graceful decommission timeout for downscaling operations. - google.protobuf.Duration graceful_decommission_timeout = 3; - - // Reasons why the Autoscaler didn't add or remove more workers. - repeated ConstrainingFactor constraints_reached = 4; - - // Less significant recommendations that are not included in the - // `AutoscalerStatus.details` message. - repeated string additional_recommendation_details = 5; - - // A unique id for this recommendation that should be included when opening - // a support ticket. - string recommendation_id = 6; - } - - // The autoscaling algorithm inputs. - Inputs inputs = 1; - - // The algorithm outputs for the recommended cluster size. - Outputs outputs = 2; -} diff --git a/third_party/googleapis/google/cloud/dataproc/v1/BUILD.bazel b/third_party/googleapis/google/cloud/dataproc/v1/BUILD.bazel deleted file mode 100644 index 5eeb456..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/BUILD.bazel +++ /dev/null @@ -1,386 +0,0 @@ -# This file was automatically generated by BuildFileGenerator - -# This is an API workspace, having public visibility by default makes perfect sense. -package(default_visibility = ["//visibility:public"]) - -############################################################################## -# Common -############################################################################## -load("@rules_proto//proto:defs.bzl", "proto_library") -load("@com_google_googleapis_imports//:imports.bzl", "proto_library_with_info") - -proto_library( - name = "dataproc_proto", - srcs = [ - "autoscaling_policies.proto", - "batches.proto", - "clusters.proto", - "jobs.proto", - "operations.proto", - "shared.proto", - "workflow_templates.proto", - ], - deps = [ - "//google/api:annotations_proto", - "//google/api:client_proto", - "//google/api:field_behavior_proto", - "//google/api:resource_proto", - "//google/longrunning:operations_proto", - "@com_google_protobuf//:duration_proto", - "@com_google_protobuf//:empty_proto", - "@com_google_protobuf//:field_mask_proto", - "@com_google_protobuf//:timestamp_proto", - ], -) - -proto_library_with_info( - name = "dataproc_proto_with_info", - deps = [ - ":dataproc_proto", - "//google/cloud:common_resources_proto", - ], -) - -############################################################################## -# Java -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "java_gapic_assembly_gradle_pkg", - "java_gapic_library", - "java_gapic_test", - "java_grpc_library", - "java_proto_library", -) - -java_proto_library( - name = "dataproc_java_proto", - deps = [":dataproc_proto"], -) - -java_grpc_library( - name = "dataproc_java_grpc", - srcs = [":dataproc_proto"], - deps = [":dataproc_java_proto"], -) - -java_gapic_library( - name = "dataproc_java_gapic", - srcs = [":dataproc_proto_with_info"], - gapic_yaml = "dataproc_gapic.yaml", - grpc_service_config = "dataproc_grpc_service_config.json", - test_deps = [ - ":dataproc_java_grpc", - ], - transport = "grpc+rest", - deps = [ - ":dataproc_java_proto", - ], -) - -java_gapic_test( - name = "dataproc_java_gapic_test_suite", - test_classes = [ - # Add AutoscalingPolicyService to gapic yaml - "com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClientHttpJsonTest", - "com.google.cloud.dataproc.v1.AutoscalingPolicyServiceClientTest", - "com.google.cloud.dataproc.v1.ClusterControllerClientHttpJsonTest", - "com.google.cloud.dataproc.v1.ClusterControllerClientTest", - "com.google.cloud.dataproc.v1.JobControllerClientHttpJsonTest", - "com.google.cloud.dataproc.v1.JobControllerClientTest", - "com.google.cloud.dataproc.v1.WorkflowTemplateServiceClientHttpJsonTest", - "com.google.cloud.dataproc.v1.WorkflowTemplateServiceClientTest", - ], - runtime_deps = [":dataproc_java_gapic_test"], -) - -# Open Source Packages -java_gapic_assembly_gradle_pkg( - name = "google-cloud-dataproc-v1-java", - include_samples = True, - transport = "grpc+rest", - deps = [ - ":dataproc_java_gapic", - ":dataproc_java_grpc", - ":dataproc_java_proto", - ":dataproc_proto", - ], -) - -############################################################################## -# Go -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "go_gapic_assembly_pkg", - "go_gapic_library", - "go_proto_library", - "go_test", -) - -go_proto_library( - name = "dataproc_go_proto", - compilers = ["@io_bazel_rules_go//proto:go_grpc"], - importpath = "google.golang.org/genproto/googleapis/cloud/dataproc/v1", - protos = [":dataproc_proto"], - deps = [ - "//google/api:annotations_go_proto", - "//google/longrunning:longrunning_go_proto", - ], -) - -go_gapic_library( - name = "dataproc_go_gapic", - srcs = [":dataproc_proto_with_info"], - grpc_service_config = "dataproc_grpc_service_config.json", - importpath = "cloud.google.com/go/dataproc/apiv1;dataproc", - service_yaml = "dataproc_v1.yaml", - transport = "grpc+rest", - deps = [ - ":dataproc_go_proto", - "//google/longrunning:longrunning_go_proto", - "@com_google_cloud_go//longrunning:go_default_library", - "@com_google_cloud_go//longrunning/autogen:go_default_library", - "@io_bazel_rules_go//proto/wkt:duration_go_proto", - ], -) - -go_test( - name = "dataproc_go_gapic_test", - srcs = [":dataproc_go_gapic_srcjar_test"], - embed = [":dataproc_go_gapic"], - importpath = "cloud.google.com/go/dataproc/apiv1", -) - -# Open Source Packages -go_gapic_assembly_pkg( - name = "gapi-cloud-dataproc-v1-go", - deps = [ - ":dataproc_go_gapic", - ":dataproc_go_gapic_srcjar-test.srcjar", - ":dataproc_go_proto", - ], -) - -############################################################################## -# Python -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "py_gapic_assembly_pkg", - "py_gapic_library", - "py_test", -) - -py_gapic_library( - name = "dataproc_py_gapic", - srcs = [":dataproc_proto"], - grpc_service_config = "dataproc_grpc_service_config.json", - transport = "grpc", -) - -py_test( - name = "dataproc_py_gapic_test", - srcs = [ - "dataproc_py_gapic_pytest.py", - "dataproc_py_gapic_test.py", - ], - legacy_create_init = False, - deps = [":dataproc_py_gapic"], -) - -py_gapic_assembly_pkg( - name = "dataproc-v1-py", - deps = [ - ":dataproc_py_gapic", - ], -) - -############################################################################## -# PHP -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "php_gapic_assembly_pkg", - "php_gapic_library", - "php_grpc_library", - "php_proto_library", -) - -php_proto_library( - name = "dataproc_php_proto", - deps = [":dataproc_proto"], -) - -php_grpc_library( - name = "dataproc_php_grpc", - srcs = [":dataproc_proto"], - deps = [":dataproc_php_proto"], -) - -php_gapic_library( - name = "dataproc_php_gapic", - srcs = [":dataproc_proto_with_info"], - gapic_yaml = "dataproc_gapic.yaml", - grpc_service_config = "dataproc_grpc_service_config.json", - service_yaml = "dataproc_v1.yaml", - deps = [ - ":dataproc_php_grpc", - ":dataproc_php_proto", - ], -) - -# Open Source Packages -php_gapic_assembly_pkg( - name = "google-cloud-dataproc-v1-php", - deps = [ - ":dataproc_php_gapic", - ":dataproc_php_grpc", - ":dataproc_php_proto", - ], -) - -############################################################################## -# Node.js -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "nodejs_gapic_assembly_pkg", - "nodejs_gapic_library", -) - -nodejs_gapic_library( - name = "dataproc_nodejs_gapic", - package_name = "@google-cloud/dataproc", - src = ":dataproc_proto_with_info", - extra_protoc_parameters = ["metadata"], - grpc_service_config = "dataproc_grpc_service_config.json", - main_service = "dataproc", - package = "google.cloud.dataproc.v1", - service_yaml = "dataproc_v1.yaml", - deps = [], -) - -nodejs_gapic_assembly_pkg( - name = "dataproc-v1-nodejs", - deps = [ - ":dataproc_nodejs_gapic", - ":dataproc_proto", - ], -) - -############################################################################## -# Ruby -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "ruby_cloud_gapic_library", - "ruby_gapic_assembly_pkg", - "ruby_grpc_library", - "ruby_proto_library", -) - -ruby_proto_library( - name = "dataproc_ruby_proto", - deps = [":dataproc_proto"], -) - -ruby_grpc_library( - name = "dataproc_ruby_grpc", - srcs = [":dataproc_proto"], - deps = [":dataproc_ruby_proto"], -) - -ruby_cloud_gapic_library( - name = "dataproc_ruby_gapic", - srcs = [":dataproc_proto_with_info"], - extra_protoc_parameters = [ - "ruby-cloud-gem-name=google-cloud-dataproc-v1", - "ruby-cloud-env-prefix=DATAPROC", - "ruby-cloud-product-url=https://cloud.google.com/dataproc", - "ruby-cloud-api-id=dataproc.googleapis.com", - "ruby-cloud-api-shortname=dataproc", - ], - grpc_service_config = "dataproc_grpc_service_config.json", - ruby_cloud_description = "Manages Hadoop-based clusters and jobs on Google Cloud Platform.", - ruby_cloud_title = "Cloud Dataproc V1", - deps = [ - ":dataproc_ruby_grpc", - ":dataproc_ruby_proto", - ], -) - -# Open Source Packages -ruby_gapic_assembly_pkg( - name = "google-cloud-dataproc-v1-ruby", - deps = [ - ":dataproc_ruby_gapic", - ":dataproc_ruby_grpc", - ":dataproc_ruby_proto", - ], -) - -############################################################################## -# C# -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "csharp_gapic_assembly_pkg", - "csharp_gapic_library", - "csharp_grpc_library", - "csharp_proto_library", -) - -csharp_proto_library( - name = "dataproc_csharp_proto", - deps = [":dataproc_proto"], -) - -csharp_grpc_library( - name = "dataproc_csharp_grpc", - srcs = [":dataproc_proto"], - deps = [":dataproc_csharp_proto"], -) - -csharp_gapic_library( - name = "dataproc_csharp_gapic", - srcs = [":dataproc_proto_with_info"], - common_resources_config = "@gax_dotnet//:Google.Api.Gax/ResourceNames/CommonResourcesConfig.json", - grpc_service_config = "dataproc_grpc_service_config.json", - service_yaml = "dataproc_v1.yaml", - deps = [ - ":dataproc_csharp_grpc", - ":dataproc_csharp_proto", - ], -) - -# Open Source Packages -csharp_gapic_assembly_pkg( - name = "google-cloud-dataproc-v1-csharp", - deps = [ - ":dataproc_csharp_gapic", - ":dataproc_csharp_grpc", - ":dataproc_csharp_proto", - ], -) - -############################################################################## -# C++ -############################################################################## -load( - "@com_google_googleapis_imports//:imports.bzl", - "cc_grpc_library", - "cc_proto_library", -) - -cc_proto_library( - name = "dataproc_cc_proto", - deps = [":dataproc_proto"], -) - -cc_grpc_library( - name = "dataproc_cc_grpc", - srcs = [":dataproc_proto"], - grpc_only = True, - deps = [":dataproc_cc_proto"], -) diff --git a/third_party/googleapis/google/cloud/dataproc/v1/autoscaling_policies.proto b/third_party/googleapis/google/cloud/dataproc/v1/autoscaling_policies.proto deleted file mode 100644 index 18b2f7d..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/autoscaling_policies.proto +++ /dev/null @@ -1,366 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.v1; - -import "google/api/annotations.proto"; -import "google/api/client.proto"; -import "google/api/field_behavior.proto"; -import "google/api/resource.proto"; -import "google/protobuf/duration.proto"; -import "google/protobuf/empty.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; -option java_multiple_files = true; -option java_outer_classname = "AutoscalingPoliciesProto"; -option java_package = "com.google.cloud.dataproc.v1"; -option (google.api.resource_definition) = { - type: "dataproc.googleapis.com/Region" - pattern: "projects/{project}/regions/{region}" -}; - -// The API interface for managing autoscaling policies in the -// Dataproc API. -service AutoscalingPolicyService { - option (google.api.default_host) = "dataproc.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; - - // Creates new autoscaling policy. - rpc CreateAutoscalingPolicy(CreateAutoscalingPolicyRequest) returns (AutoscalingPolicy) { - option (google.api.http) = { - post: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies" - body: "policy" - additional_bindings { - post: "/v1/{parent=projects/*/regions/*}/autoscalingPolicies" - body: "policy" - } - }; - option (google.api.method_signature) = "parent,policy"; - } - - // Updates (replaces) autoscaling policy. - // - // Disabled check for update_mask, because all updates will be full - // replacements. - rpc UpdateAutoscalingPolicy(UpdateAutoscalingPolicyRequest) returns (AutoscalingPolicy) { - option (google.api.http) = { - put: "/v1/{policy.name=projects/*/locations/*/autoscalingPolicies/*}" - body: "policy" - additional_bindings { - put: "/v1/{policy.name=projects/*/regions/*/autoscalingPolicies/*}" - body: "policy" - } - }; - option (google.api.method_signature) = "policy"; - } - - // Retrieves autoscaling policy. - rpc GetAutoscalingPolicy(GetAutoscalingPolicyRequest) returns (AutoscalingPolicy) { - option (google.api.http) = { - get: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}" - additional_bindings { - get: "/v1/{name=projects/*/regions/*/autoscalingPolicies/*}" - } - }; - option (google.api.method_signature) = "name"; - } - - // Lists autoscaling policies in the project. - rpc ListAutoscalingPolicies(ListAutoscalingPoliciesRequest) returns (ListAutoscalingPoliciesResponse) { - option (google.api.http) = { - get: "/v1/{parent=projects/*/locations/*}/autoscalingPolicies" - additional_bindings { - get: "/v1/{parent=projects/*/regions/*}/autoscalingPolicies" - } - }; - option (google.api.method_signature) = "parent"; - } - - // Deletes an autoscaling policy. It is an error to delete an autoscaling - // policy that is in use by one or more clusters. - rpc DeleteAutoscalingPolicy(DeleteAutoscalingPolicyRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { - delete: "/v1/{name=projects/*/locations/*/autoscalingPolicies/*}" - additional_bindings { - delete: "/v1/{name=projects/*/regions/*/autoscalingPolicies/*}" - } - }; - option (google.api.method_signature) = "name"; - } -} - -// Describes an autoscaling policy for Dataproc cluster autoscaler. -message AutoscalingPolicy { - option (google.api.resource) = { - type: "dataproc.googleapis.com/AutoscalingPolicy" - pattern: "projects/{project}/locations/{location}/autoscalingPolicies/{autoscaling_policy}" - pattern: "projects/{project}/regions/{region}/autoscalingPolicies/{autoscaling_policy}" - }; - - // Required. The policy id. - // - // The id must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). Cannot begin or end with underscore - // or hyphen. Must consist of between 3 and 50 characters. - // - string id = 1; - - // Output only. The "resource name" of the autoscaling policy, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.autoscalingPolicies`, the resource name of the - // policy has the following format: - // `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}` - // - // * For `projects.locations.autoscalingPolicies`, the resource name of the - // policy has the following format: - // `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}` - string name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Autoscaling algorithm for policy. - oneof algorithm { - BasicAutoscalingAlgorithm basic_algorithm = 3 [(google.api.field_behavior) = REQUIRED]; - } - - // Required. Describes how the autoscaler will operate for primary workers. - InstanceGroupAutoscalingPolicyConfig worker_config = 4 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Describes how the autoscaler will operate for secondary workers. - InstanceGroupAutoscalingPolicyConfig secondary_worker_config = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The labels to associate with this autoscaling policy. - // Label **keys** must contain 1 to 63 characters, and must conform to - // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt). - // Label **values** may be empty, but, if present, must contain 1 to 63 - // characters, and must conform to [RFC - // 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be - // associated with an autoscaling policy. - map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// Basic algorithm for autoscaling. -message BasicAutoscalingAlgorithm { - oneof config { - // Required. YARN autoscaling configuration. - BasicYarnAutoscalingConfig yarn_config = 1 [(google.api.field_behavior) = REQUIRED]; - } - - // Optional. Duration between scaling events. A scaling period starts after - // the update operation from the previous event has completed. - // - // Bounds: [2m, 1d]. Default: 2m. - google.protobuf.Duration cooldown_period = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// Basic autoscaling configurations for YARN. -message BasicYarnAutoscalingConfig { - // Required. Timeout for YARN graceful decommissioning of Node Managers. - // Specifies the duration to wait for jobs to complete before forcefully - // removing workers (and potentially interrupting jobs). Only applicable to - // downscaling operations. - // - // Bounds: [0s, 1d]. - google.protobuf.Duration graceful_decommission_timeout = 5 [(google.api.field_behavior) = REQUIRED]; - - // Required. Fraction of average YARN pending memory in the last cooldown period - // for which to add workers. A scale-up factor of 1.0 will result in scaling - // up so that there is no pending memory remaining after the update (more - // aggressive scaling). A scale-up factor closer to 0 will result in a smaller - // magnitude of scaling up (less aggressive scaling). - // See [How autoscaling - // works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works) - // for more information. - // - // Bounds: [0.0, 1.0]. - double scale_up_factor = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. Fraction of average YARN pending memory in the last cooldown period - // for which to remove workers. A scale-down factor of 1 will result in - // scaling down so that there is no available memory remaining after the - // update (more aggressive scaling). A scale-down factor of 0 disables - // removing workers, which can be beneficial for autoscaling a single job. - // See [How autoscaling - // works](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/autoscaling#how_autoscaling_works) - // for more information. - // - // Bounds: [0.0, 1.0]. - double scale_down_factor = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Minimum scale-up threshold as a fraction of total cluster size - // before scaling occurs. For example, in a 20-worker cluster, a threshold of - // 0.1 means the autoscaler must recommend at least a 2-worker scale-up for - // the cluster to scale. A threshold of 0 means the autoscaler will scale up - // on any recommended change. - // - // Bounds: [0.0, 1.0]. Default: 0.0. - double scale_up_min_worker_fraction = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Minimum scale-down threshold as a fraction of total cluster size - // before scaling occurs. For example, in a 20-worker cluster, a threshold of - // 0.1 means the autoscaler must recommend at least a 2 worker scale-down for - // the cluster to scale. A threshold of 0 means the autoscaler will scale down - // on any recommended change. - // - // Bounds: [0.0, 1.0]. Default: 0.0. - double scale_down_min_worker_fraction = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// Configuration for the size bounds of an instance group, including its -// proportional size to other groups. -message InstanceGroupAutoscalingPolicyConfig { - // Optional. Minimum number of instances for this group. - // - // Primary workers - Bounds: [2, max_instances]. Default: 2. - // Secondary workers - Bounds: [0, max_instances]. Default: 0. - int32 min_instances = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Required. Maximum number of instances for this group. Required for primary - // workers. Note that by default, clusters will not use secondary workers. - // Required for secondary workers if the minimum secondary instances is set. - // - // Primary workers - Bounds: [min_instances, ). - // Secondary workers - Bounds: [min_instances, ). Default: 0. - int32 max_instances = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Weight for the instance group, which is used to determine the - // fraction of total workers in the cluster from this instance group. - // For example, if primary workers have weight 2, and secondary workers have - // weight 1, the cluster will have approximately 2 primary workers for each - // secondary worker. - // - // The cluster may not reach the specified balance if constrained - // by min/max bounds or other autoscaling settings. For example, if - // `max_instances` for secondary workers is 0, then only primary workers will - // be added. The cluster can also be out of balance when created. - // - // If weight is not set on any instance group, the cluster will default to - // equal weight for all groups: the cluster will attempt to maintain an equal - // number of workers in each group within the configured size bounds for each - // group. If weight is set for one group only, the cluster will default to - // zero weight on the unset group. For example if weight is set only on - // primary workers, the cluster will use primary workers only and no - // secondary workers. - int32 weight = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to create an autoscaling policy. -message CreateAutoscalingPolicyRequest { - // Required. The "resource name" of the region or location, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.autoscalingPolicies.create`, the resource name - // of the region has the following format: - // `projects/{project_id}/regions/{region}` - // - // * For `projects.locations.autoscalingPolicies.create`, the resource name - // of the location has the following format: - // `projects/{project_id}/locations/{location}` - string parent = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - child_type: "dataproc.googleapis.com/AutoscalingPolicy" - } - ]; - - // Required. The autoscaling policy to create. - AutoscalingPolicy policy = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to fetch an autoscaling policy. -message GetAutoscalingPolicyRequest { - // Required. The "resource name" of the autoscaling policy, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.autoscalingPolicies.get`, the resource name - // of the policy has the following format: - // `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}` - // - // * For `projects.locations.autoscalingPolicies.get`, the resource name - // of the policy has the following format: - // `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}` - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dataproc.googleapis.com/AutoscalingPolicy" - } - ]; -} - -// A request to update an autoscaling policy. -message UpdateAutoscalingPolicyRequest { - // Required. The updated autoscaling policy. - AutoscalingPolicy policy = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to delete an autoscaling policy. -// -// Autoscaling policies in use by one or more clusters will not be deleted. -message DeleteAutoscalingPolicyRequest { - // Required. The "resource name" of the autoscaling policy, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.autoscalingPolicies.delete`, the resource name - // of the policy has the following format: - // `projects/{project_id}/regions/{region}/autoscalingPolicies/{policy_id}` - // - // * For `projects.locations.autoscalingPolicies.delete`, the resource name - // of the policy has the following format: - // `projects/{project_id}/locations/{location}/autoscalingPolicies/{policy_id}` - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dataproc.googleapis.com/AutoscalingPolicy" - } - ]; -} - -// A request to list autoscaling policies in a project. -message ListAutoscalingPoliciesRequest { - // Required. The "resource name" of the region or location, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.autoscalingPolicies.list`, the resource name - // of the region has the following format: - // `projects/{project_id}/regions/{region}` - // - // * For `projects.locations.autoscalingPolicies.list`, the resource name - // of the location has the following format: - // `projects/{project_id}/locations/{location}` - string parent = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - child_type: "dataproc.googleapis.com/AutoscalingPolicy" - } - ]; - - // Optional. The maximum number of results to return in each response. - // Must be less than or equal to 1000. Defaults to 100. - int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The page token, returned by a previous call, to request the - // next page of results. - string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// A response to a request to list autoscaling policies in a project. -message ListAutoscalingPoliciesResponse { - // Output only. Autoscaling policies list. - repeated AutoscalingPolicy policies = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. This token is included in the response if there are more - // results to fetch. - string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; -} diff --git a/third_party/googleapis/google/cloud/dataproc/v1/batches.proto b/third_party/googleapis/google/cloud/dataproc/v1/batches.proto deleted file mode 100644 index eafb4e3..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/batches.proto +++ /dev/null @@ -1,372 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.v1; - -import "google/api/annotations.proto"; -import "google/api/client.proto"; -import "google/api/field_behavior.proto"; -import "google/api/resource.proto"; -import "google/cloud/dataproc/v1/shared.proto"; -import "google/longrunning/operations.proto"; -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; -option java_multiple_files = true; -option java_outer_classname = "BatchesProto"; -option java_package = "com.google.cloud.dataproc.v1"; - -// The BatchController provides methods to manage batch workloads. -service BatchController { - option (google.api.default_host) = "dataproc.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; - - // Creates a batch workload that executes asynchronously. - rpc CreateBatch(CreateBatchRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/{parent=projects/*/locations/*}/batches" - body: "batch" - }; - option (google.api.method_signature) = "parent,batch,batch_id"; - option (google.longrunning.operation_info) = { - response_type: "Batch" - metadata_type: "google.cloud.dataproc.v1.BatchOperationMetadata" - }; - } - - // Gets the batch workload resource representation. - rpc GetBatch(GetBatchRequest) returns (Batch) { - option (google.api.http) = { - get: "/v1/{name=projects/*/locations/*/batches/*}" - }; - option (google.api.method_signature) = "name"; - } - - // Lists batch workloads. - rpc ListBatches(ListBatchesRequest) returns (ListBatchesResponse) { - option (google.api.http) = { - get: "/v1/{parent=projects/*/locations/*}/batches" - }; - option (google.api.method_signature) = "parent"; - } - - // Deletes the batch workload resource. If the batch is not in terminal state, - // the delete fails and the response returns `FAILED_PRECONDITION`. - rpc DeleteBatch(DeleteBatchRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { - delete: "/v1/{name=projects/*/locations/*/batches/*}" - }; - option (google.api.method_signature) = "name"; - } -} - -// A request to create a batch workload. -message CreateBatchRequest { - // Required. The parent resource where this batch will be created. - string parent = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - child_type: "dataproc.googleapis.com/Batch" - } - ]; - - // Required. The batch to create. - Batch batch = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The ID to use for the batch, which will become the final component of - // the batch's resource name. - // - // This value must be 4-63 characters. Valid characters are `/[a-z][0-9]-/`. - string batch_id = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A unique ID used to identify the request. If the service - // receives two - // [CreateBatchRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.CreateBatchRequest)s - // with the same request_id, the second request is ignored and the - // Operation that corresponds to the first Batch created and stored - // in the backend is returned. - // - // Recommendation: Set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The value must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to get the resource representation for a batch workload. -message GetBatchRequest { - // Required. The name of the batch to retrieve. - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dataproc.googleapis.com/Batch" - } - ]; -} - -// A request to list batch workloads in a project. -message ListBatchesRequest { - // Required. The parent, which owns this collection of batches. - string parent = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - child_type: "dataproc.googleapis.com/Batch" - } - ]; - - // Optional. The maximum number of batches to return in each response. - // The service may return fewer than this value. - // The default page size is 20; the maximum page size is 1000. - int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A page token received from a previous `ListBatches` call. - // Provide this token to retrieve the subsequent page. - string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// A list of batch workloads. -message ListBatchesResponse { - // The batches from the specified collection. - repeated Batch batches = 1; - - // A token, which can be sent as `page_token` to retrieve the next page. - // If this field is omitted, there are no subsequent pages. - string next_page_token = 2; -} - -// A request to delete a batch workload. -message DeleteBatchRequest { - // Required. The name of the batch resource to delete. - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dataproc.googleapis.com/Batch" - } - ]; -} - -// A representation of a batch workload in the service. -message Batch { - option (google.api.resource) = { - type: "dataproc.googleapis.com/Batch" - pattern: "projects/{project}/locations/{location}/batches/{batch}" - }; - - // Historical state information. - message StateHistory { - // Output only. The state of the batch at this point in history. - State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Details about the state at this point in history. - string state_message = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The time when the batch entered the historical state. - google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - } - - // The batch state. - enum State { - // The batch state is unknown. - STATE_UNSPECIFIED = 0; - - // The batch is created before running. - PENDING = 1; - - // The batch is running. - RUNNING = 2; - - // The batch is cancelling. - CANCELLING = 3; - - // The batch cancellation was successful. - CANCELLED = 4; - - // The batch completed successfully. - SUCCEEDED = 5; - - // The batch is no longer running due to an error. - FAILED = 6; - } - - // Output only. The resource name of the batch. - string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. A batch UUID (Unique Universal Identifier). The service - // generates this value when it creates the batch. - string uuid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The time when the batch was created. - google.protobuf.Timestamp create_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // The application/framework-specific portion of the batch configuration. - oneof batch_config { - // Optional. PySpark batch config. - PySparkBatch pyspark_batch = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Spark batch config. - SparkBatch spark_batch = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. SparkR batch config. - SparkRBatch spark_r_batch = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. SparkSql batch config. - SparkSqlBatch spark_sql_batch = 7 [(google.api.field_behavior) = OPTIONAL]; - } - - // Output only. Runtime information about batch execution. - RuntimeInfo runtime_info = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The state of the batch. - State state = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Batch state details, such as a failure - // description if the state is `FAILED`. - string state_message = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The time when the batch entered a current state. - google.protobuf.Timestamp state_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The email address of the user who created the batch. - string creator = 12 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. The labels to associate with this batch. - // Label **keys** must contain 1 to 63 characters, and must conform to - // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt). - // Label **values** may be empty, but, if present, must contain 1 to 63 - // characters, and must conform to [RFC - // 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be - // associated with a batch. - map<string, string> labels = 13 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Runtime configuration for the batch execution. - RuntimeConfig runtime_config = 14 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Environment configuration for the batch execution. - EnvironmentConfig environment_config = 15 [(google.api.field_behavior) = OPTIONAL]; - - // Output only. The resource name of the operation associated with this batch. - string operation = 16 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Historical state information for the batch. - repeated StateHistory state_history = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// A configuration for running an -// [Apache -// PySpark](https://spark.apache.org/docs/latest/api/python/getting_started/quickstart.html) -// batch workload. -message PySparkBatch { - // Required. The HCFS URI of the main Python file to use as the Spark driver. Must - // be a .py file. - string main_python_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The arguments to pass to the driver. Do not include arguments - // that can be set as batch properties, such as `--conf`, since a collision - // can occur that causes an incorrect batch submission. - repeated string args = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS file URIs of Python files to pass to the PySpark - // framework. Supported file types: `.py`, `.egg`, and `.zip`. - repeated string python_file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the classpath of the - // Spark driver and tasks. - repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. - repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // `.jar`, `.tar`, `.tar.gz`, `.tgz`, and `.zip`. - repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A configuration for running an [Apache Spark](http://spark.apache.org/) -// batch workload. -message SparkBatch { - // The specification of the main method to call to drive the Spark - // workload. Specify either the jar file that contains the main class or the - // main class name. To pass both a main jar and a main class in that jar, add - // the jar to `jar_file_uris`, and then specify the main class - // name in `main_class`. - oneof driver { - // Optional. The HCFS URI of the jar file that contains the main class. - string main_jar_file_uri = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The name of the driver main class. The jar file that contains the class - // must be in the classpath or specified in `jar_file_uris`. - string main_class = 2 [(google.api.field_behavior) = OPTIONAL]; - } - - // Optional. The arguments to pass to the driver. Do not include arguments - // that can be set as batch properties, such as `--conf`, since a collision - // can occur that causes an incorrect batch submission. - repeated string args = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the classpath of the - // Spark driver and tasks. - repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. - repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // `.jar`, `.tar`, `.tar.gz`, `.tgz`, and `.zip`. - repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A configuration for running an -// [Apache SparkR](https://spark.apache.org/docs/latest/sparkr.html) -// batch workload. -message SparkRBatch { - // Required. The HCFS URI of the main R file to use as the driver. - // Must be a `.R` or `.r` file. - string main_r_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The arguments to pass to the Spark driver. Do not include arguments - // that can be set as batch properties, such as `--conf`, since a collision - // can occur that causes an incorrect batch submission. - repeated string args = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. - repeated string file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // `.jar`, `.tar`, `.tar.gz`, `.tgz`, and `.zip`. - repeated string archive_uris = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// A configuration for running -// [Apache Spark SQL](http://spark.apache.org/sql/) queries as a batch workload. -message SparkSqlBatch { - // Required. The HCFS URI of the script that contains Spark SQL queries to execute. - string query_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Mapping of query variable names to values (equivalent to the - // Spark SQL command: `SET name="value";`). - map<string, string> query_variables = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH. - repeated string jar_file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; -} diff --git a/third_party/googleapis/google/cloud/dataproc/v1/clusters.proto b/third_party/googleapis/google/cloud/dataproc/v1/clusters.proto deleted file mode 100644 index 9550278..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/clusters.proto +++ /dev/null @@ -1,1359 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.v1; - -import "google/api/annotations.proto"; -import "google/api/client.proto"; -import "google/api/field_behavior.proto"; -import "google/api/resource.proto"; -import "google/cloud/dataproc/v1/shared.proto"; -import "google/longrunning/operations.proto"; -import "google/protobuf/duration.proto"; -import "google/protobuf/field_mask.proto"; -import "google/protobuf/timestamp.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; -option java_multiple_files = true; -option java_outer_classname = "ClustersProto"; -option java_package = "com.google.cloud.dataproc.v1"; -option (google.api.resource_definition) = { - type: "container.googleapis.com/Cluster" - pattern: "projects/{project}/locations/{location}/clusters/{cluster}" -}; -option (google.api.resource_definition) = { - type: "metastore.googleapis.com/Service" - pattern: "projects/{project}/locations/{location}/services/{service}" -}; - -// The ClusterControllerService provides methods to manage clusters -// of Compute Engine instances. -service ClusterController { - option (google.api.default_host) = "dataproc.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; - - // Creates a cluster in a project. The returned - // [Operation.metadata][google.longrunning.Operation.metadata] will be - // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). - rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/clusters" - body: "cluster" - }; - option (google.api.method_signature) = "project_id,region,cluster"; - option (google.longrunning.operation_info) = { - response_type: "Cluster" - metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" - }; - } - - // Updates a cluster in a project. The returned - // [Operation.metadata][google.longrunning.Operation.metadata] will be - // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). - // The cluster must be in a [`RUNNING`][google.cloud.dataproc.v1.ClusterStatus.State] state or an error - // is returned. - rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" - body: "cluster" - }; - option (google.api.method_signature) = "project_id,region,cluster_name,cluster,update_mask"; - option (google.longrunning.operation_info) = { - response_type: "Cluster" - metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" - }; - } - - // Stops a cluster in a project. - rpc StopCluster(StopClusterRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:stop" - body: "*" - }; - option (google.longrunning.operation_info) = { - response_type: "Cluster" - metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" - }; - } - - // Starts a cluster in a project. - rpc StartCluster(StartClusterRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:start" - body: "*" - }; - option (google.longrunning.operation_info) = { - response_type: "Cluster" - metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" - }; - } - - // Deletes a cluster in a project. The returned - // [Operation.metadata][google.longrunning.Operation.metadata] will be - // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). - rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" - }; - option (google.api.method_signature) = "project_id,region,cluster_name"; - option (google.longrunning.operation_info) = { - response_type: "google.protobuf.Empty" - metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" - }; - } - - // Gets the resource representation for a cluster in a project. - rpc GetCluster(GetClusterRequest) returns (Cluster) { - option (google.api.http) = { - get: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" - }; - option (google.api.method_signature) = "project_id,region,cluster_name"; - } - - // Lists all regions/{region}/clusters in a project alphabetically. - rpc ListClusters(ListClustersRequest) returns (ListClustersResponse) { - option (google.api.http) = { - get: "/v1/projects/{project_id}/regions/{region}/clusters" - }; - option (google.api.method_signature) = "project_id,region"; - option (google.api.method_signature) = "project_id,region,filter"; - } - - // Gets cluster diagnostic information. The returned - // [Operation.metadata][google.longrunning.Operation.metadata] will be - // [ClusterOperationMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata). - // After the operation completes, - // [Operation.response][google.longrunning.Operation.response] - // contains - // [DiagnoseClusterResults](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults). - rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose" - body: "*" - }; - option (google.api.method_signature) = "project_id,region,cluster_name"; - option (google.longrunning.operation_info) = { - response_type: "DiagnoseClusterResults" - metadata_type: "google.cloud.dataproc.v1.ClusterOperationMetadata" - }; - } -} - -// Describes the identifying information, config, and status of -// a Dataproc cluster -message Cluster { - // Required. The Google Cloud Platform project ID that the cluster belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster name. Cluster names within a project must be - // unique. Names of deleted clusters can be reused. - string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The cluster config for a cluster of Compute Engine Instances. - // Note that Dataproc may set default values, and values may change - // when clusters are updated. - ClusterConfig config = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The virtual cluster config, used when creating a Dataproc cluster that - // does not directly control the underlying compute resources, for example, - // when creating a [Dataproc-on-GKE - // cluster](https://cloud.google.com/dataproc/docs/concepts/jobs/dataproc-gke#create-a-dataproc-on-gke-cluster). - // Note that Dataproc may set default values, and values may change when - // clusters are updated. Exactly one of config or virtualClusterConfig must be - // specified. - VirtualClusterConfig virtual_cluster_config = 10 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The labels to associate with this cluster. - // Label **keys** must contain 1 to 63 characters, and must conform to - // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt). - // Label **values** may be empty, but, if present, must contain 1 to 63 - // characters, and must conform to [RFC - // 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be - // associated with a cluster. - map<string, string> labels = 8 [(google.api.field_behavior) = OPTIONAL]; - - // Output only. Cluster status. - ClusterStatus status = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The previous cluster status. - repeated ClusterStatus status_history = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. A cluster UUID (Unique Universal Identifier). Dataproc - // generates this value when it creates the cluster. - string cluster_uuid = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Contains cluster daemon metrics such as HDFS and YARN stats. - // - // **Beta Feature**: This report is available for testing purposes only. It - // may be changed before final release. - ClusterMetrics metrics = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// The cluster config. -message ClusterConfig { - // Optional. A Cloud Storage bucket used to stage job - // dependencies, config files, and job driver console output. - // If you do not specify a staging bucket, Cloud - // Dataproc will determine a Cloud Storage location (US, - // ASIA, or EU) for your cluster's staging bucket according to the - // Compute Engine zone where your cluster is deployed, and then create - // and manage this project-level, per-location bucket (see - // [Dataproc staging and temp - // buckets](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)). - // **This field requires a Cloud Storage bucket name, not a `gs://...` URI to - // a Cloud Storage bucket.** - string config_bucket = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A Cloud Storage bucket used to store ephemeral cluster and jobs data, - // such as Spark and MapReduce history files. - // If you do not specify a temp bucket, - // Dataproc will determine a Cloud Storage location (US, - // ASIA, or EU) for your cluster's temp bucket according to the - // Compute Engine zone where your cluster is deployed, and then create - // and manage this project-level, per-location bucket. The default bucket has - // a TTL of 90 days, but you can use any TTL (or none) if you specify a - // bucket (see - // [Dataproc staging and temp - // buckets](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)). - // **This field requires a Cloud Storage bucket name, not a `gs://...` URI to - // a Cloud Storage bucket.** - string temp_bucket = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The shared Compute Engine config settings for - // all instances in a cluster. - GceClusterConfig gce_cluster_config = 8 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Compute Engine config settings for - // the cluster's master instance. - InstanceGroupConfig master_config = 9 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Compute Engine config settings for - // the cluster's worker instances. - InstanceGroupConfig worker_config = 10 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Compute Engine config settings for - // a cluster's secondary worker instances - InstanceGroupConfig secondary_worker_config = 12 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The config settings for cluster software. - SoftwareConfig software_config = 13 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Commands to execute on each node after config is - // completed. By default, executables are run on master and all worker nodes. - // You can test a node's `role` metadata to run an executable on - // a master or worker node, as shown below using `curl` (you can also use - // `wget`): - // - // ROLE=$(curl -H Metadata-Flavor:Google - // http://metadata/computeMetadata/v1/instance/attributes/dataproc-role) - // if [[ "${ROLE}" == 'Master' ]]; then - // ... master specific actions ... - // else - // ... worker specific actions ... - // fi - repeated NodeInitializationAction initialization_actions = 11 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Encryption settings for the cluster. - EncryptionConfig encryption_config = 15 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Autoscaling config for the policy associated with the cluster. - // Cluster does not autoscale if this field is unset. - AutoscalingConfig autoscaling_config = 18 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Security settings for the cluster. - SecurityConfig security_config = 16 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Lifecycle setting for the cluster. - LifecycleConfig lifecycle_config = 17 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Port/endpoint configuration for this cluster - EndpointConfig endpoint_config = 19 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Metastore configuration. - MetastoreConfig metastore_config = 20 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The config for Dataproc metrics. - DataprocMetricConfig dataproc_metric_config = 23 [(google.api.field_behavior) = OPTIONAL]; -} - -// Dataproc cluster config for a cluster that does not directly control the -// underlying compute resources, such as a [Dataproc-on-GKE -// cluster](https://cloud.google.com/dataproc/docs/concepts/jobs/dataproc-gke#create-a-dataproc-on-gke-cluster). -message VirtualClusterConfig { - // Optional. A Storage bucket used to stage job - // dependencies, config files, and job driver console output. - // If you do not specify a staging bucket, Cloud - // Dataproc will determine a Cloud Storage location (US, - // ASIA, or EU) for your cluster's staging bucket according to the - // Compute Engine zone where your cluster is deployed, and then create - // and manage this project-level, per-location bucket (see - // [Dataproc staging and temp - // buckets](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/staging-bucket)). - // **This field requires a Cloud Storage bucket name, not a `gs://...` URI to - // a Cloud Storage bucket.** - string staging_bucket = 1 [(google.api.field_behavior) = OPTIONAL]; - - oneof infrastructure_config { - // Required. The configuration for running the Dataproc cluster on Kubernetes. - KubernetesClusterConfig kubernetes_cluster_config = 6 [(google.api.field_behavior) = REQUIRED]; - } - - // Optional. Configuration of auxiliary services used by this cluster. - AuxiliaryServicesConfig auxiliary_services_config = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// Auxiliary services configuration for a Cluster. -message AuxiliaryServicesConfig { - // Optional. The Hive Metastore configuration for this workload. - MetastoreConfig metastore_config = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Spark History Server configuration for the workload. - SparkHistoryServerConfig spark_history_server_config = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// Endpoint config for this cluster -message EndpointConfig { - // Output only. The map of port descriptions to URLs. Will only be populated - // if enable_http_port_access is true. - map<string, string> http_ports = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. If true, enable http access to specific ports on the cluster - // from external sources. Defaults to false. - bool enable_http_port_access = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// Autoscaling Policy config associated with the cluster. -message AutoscalingConfig { - // Optional. The autoscaling policy used by the cluster. - // - // Only resource names including projectid and location (region) are valid. - // Examples: - // - // * `https://www.googleapis.com/compute/v1/projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]` - // * `projects/[project_id]/locations/[dataproc_region]/autoscalingPolicies/[policy_id]` - // - // Note that the policy must be in the same project and Dataproc region. - string policy_uri = 1 [(google.api.field_behavior) = OPTIONAL]; -} - -// Encryption settings for the cluster. -message EncryptionConfig { - // Optional. The Cloud KMS key name to use for PD disk encryption for all - // instances in the cluster. - string gce_pd_kms_key_name = 1 [(google.api.field_behavior) = OPTIONAL]; -} - -// Common config settings for resources of Compute Engine cluster -// instances, applicable to all instances in the cluster. -message GceClusterConfig { - // `PrivateIpv6GoogleAccess` controls whether and how Dataproc cluster nodes - // can communicate with Google Services through gRPC over IPv6. - // These values are directly mapped to corresponding values in the - // [Compute Engine Instance - // fields](https://cloud.google.com/compute/docs/reference/rest/v1/instances). - enum PrivateIpv6GoogleAccess { - // If unspecified, Compute Engine default behavior will apply, which - // is the same as [INHERIT_FROM_SUBNETWORK][google.cloud.dataproc.v1.GceClusterConfig.PrivateIpv6GoogleAccess.INHERIT_FROM_SUBNETWORK]. - PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED = 0; - - // Private access to and from Google Services configuration - // inherited from the subnetwork configuration. This is the - // default Compute Engine behavior. - INHERIT_FROM_SUBNETWORK = 1; - - // Enables outbound private IPv6 access to Google Services from the Dataproc - // cluster. - OUTBOUND = 2; - - // Enables bidirectional private IPv6 access between Google Services and the - // Dataproc cluster. - BIDIRECTIONAL = 3; - } - - // Optional. The zone where the Compute Engine cluster will be located. - // On a create request, it is required in the "global" region. If omitted - // in a non-global Dataproc region, the service will pick a zone in the - // corresponding Compute Engine region. On a get request, zone will - // always be present. - // - // A full URL, partial URI, or short name are valid. Examples: - // - // * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]` - // * `projects/[project_id]/zones/[zone]` - // * `us-central1-f` - string zone_uri = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Compute Engine network to be used for machine - // communications. Cannot be specified with subnetwork_uri. If neither - // `network_uri` nor `subnetwork_uri` is specified, the "default" network of - // the project is used, if it exists. Cannot be a "Custom Subnet Network" (see - // [Using Subnetworks](https://cloud.google.com/compute/docs/subnetworks) for - // more information). - // - // A full URL, partial URI, or short name are valid. Examples: - // - // * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/global/default` - // * `projects/[project_id]/regions/global/default` - // * `default` - string network_uri = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Compute Engine subnetwork to be used for machine - // communications. Cannot be specified with network_uri. - // - // A full URL, partial URI, or short name are valid. Examples: - // - // * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/subnetworks/sub0` - // * `projects/[project_id]/regions/us-east1/subnetworks/sub0` - // * `sub0` - string subnetwork_uri = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. If true, all instances in the cluster will only have internal IP - // addresses. By default, clusters are not restricted to internal IP - // addresses, and will have ephemeral external IP addresses assigned to each - // instance. This `internal_ip_only` restriction can only be enabled for - // subnetwork enabled networks, and all off-cluster dependencies must be - // configured to be accessible without external IP addresses. - bool internal_ip_only = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The type of IPv6 access for a cluster. - PrivateIpv6GoogleAccess private_ipv6_google_access = 12 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The [Dataproc service - // account](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#service_accounts_in_dataproc) - // (also see [VM Data Plane - // identity](https://cloud.google.com/dataproc/docs/concepts/iam/dataproc-principals#vm_service_account_data_plane_identity)) - // used by Dataproc cluster VM instances to access Google Cloud Platform - // services. - // - // If not specified, the - // [Compute Engine default service - // account](https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) - // is used. - string service_account = 8 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The URIs of service account scopes to be included in - // Compute Engine instances. The following base set of scopes is always - // included: - // - // * https://www.googleapis.com/auth/cloud.useraccounts.readonly - // * https://www.googleapis.com/auth/devstorage.read_write - // * https://www.googleapis.com/auth/logging.write - // - // If no scopes are specified, the following defaults are also provided: - // - // * https://www.googleapis.com/auth/bigquery - // * https://www.googleapis.com/auth/bigtable.admin.table - // * https://www.googleapis.com/auth/bigtable.data - // * https://www.googleapis.com/auth/devstorage.full_control - repeated string service_account_scopes = 3 [(google.api.field_behavior) = OPTIONAL]; - - // The Compute Engine tags to add to all instances (see [Tagging - // instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)). - repeated string tags = 4; - - // The Compute Engine metadata entries to add to all instances (see - // [Project and instance - // metadata](https://cloud.google.com/compute/docs/storing-retrieving-metadata#project_and_instance_metadata)). - map<string, string> metadata = 5; - - // Optional. Reservation Affinity for consuming Zonal reservation. - ReservationAffinity reservation_affinity = 11 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Node Group Affinity for sole-tenant clusters. - NodeGroupAffinity node_group_affinity = 13 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Shielded Instance Config for clusters using [Compute Engine Shielded - // VMs](https://cloud.google.com/security/shielded-cloud/shielded-vm). - ShieldedInstanceConfig shielded_instance_config = 14 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Confidential Instance Config for clusters using [Confidential - // VMs](https://cloud.google.com/compute/confidential-vm/docs). - ConfidentialInstanceConfig confidential_instance_config = 15 [(google.api.field_behavior) = OPTIONAL]; -} - -// Node Group Affinity for clusters using sole-tenant node groups. -message NodeGroupAffinity { - // Required. The URI of a - // sole-tenant [node group - // resource](https://cloud.google.com/compute/docs/reference/rest/v1/nodeGroups) - // that the cluster will be created on. - // - // A full URL, partial URI, or node group name are valid. Examples: - // - // * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-central1-a/nodeGroups/node-group-1` - // * `projects/[project_id]/zones/us-central1-a/nodeGroups/node-group-1` - // * `node-group-1` - string node_group_uri = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// Shielded Instance Config for clusters using [Compute Engine Shielded -// VMs](https://cloud.google.com/security/shielded-cloud/shielded-vm). -message ShieldedInstanceConfig { - // Optional. Defines whether instances have Secure Boot enabled. - bool enable_secure_boot = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Defines whether instances have the vTPM enabled. - bool enable_vtpm = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Defines whether instances have integrity monitoring enabled. - bool enable_integrity_monitoring = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// Confidential Instance Config for clusters using [Confidential -// VMs](https://cloud.google.com/compute/confidential-vm/docs) -message ConfidentialInstanceConfig { - // Optional. Defines whether the instance should have confidential compute enabled. - bool enable_confidential_compute = 1 [(google.api.field_behavior) = OPTIONAL]; -} - -// The config settings for Compute Engine resources in -// an instance group, such as a master or worker group. -message InstanceGroupConfig { - // Controls the use of - // [preemptible instances] - // (https://cloud.google.com/compute/docs/instances/preemptible) - // within the group. - enum Preemptibility { - // Preemptibility is unspecified, the system will choose the - // appropriate setting for each instance group. - PREEMPTIBILITY_UNSPECIFIED = 0; - - // Instances are non-preemptible. - // - // This option is allowed for all instance groups and is the only valid - // value for Master and Worker instance groups. - NON_PREEMPTIBLE = 1; - - // Instances are preemptible. - // - // This option is allowed only for secondary worker groups. - PREEMPTIBLE = 2; - } - - // Optional. The number of VM instances in the instance group. - // For [HA - // cluster](/dataproc/docs/concepts/configuring-clusters/high-availability) - // [master_config](#FIELDS.master_config) groups, **must be set to 3**. - // For standard cluster [master_config](#FIELDS.master_config) groups, - // **must be set to 1**. - int32 num_instances = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Output only. The list of instance names. Dataproc derives the names - // from `cluster_name`, `num_instances`, and the instance group. - repeated string instance_names = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. The Compute Engine image resource used for cluster instances. - // - // The URI can represent an image or image family. - // - // Image examples: - // - // * `https://www.googleapis.com/compute/beta/projects/[project_id]/global/images/[image-id]` - // * `projects/[project_id]/global/images/[image-id]` - // * `image-id` - // - // Image family examples. Dataproc will use the most recent - // image from the family: - // - // * `https://www.googleapis.com/compute/beta/projects/[project_id]/global/images/family/[custom-image-family-name]` - // * `projects/[project_id]/global/images/family/[custom-image-family-name]` - // - // If the URI is unspecified, it will be inferred from - // `SoftwareConfig.image_version` or the system default. - string image_uri = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Compute Engine machine type used for cluster instances. - // - // A full URL, partial URI, or short name are valid. Examples: - // - // * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2` - // * `projects/[project_id]/zones/us-east1-a/machineTypes/n1-standard-2` - // * `n1-standard-2` - // - // **Auto Zone Exception**: If you are using the Dataproc - // [Auto Zone - // Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) - // feature, you must use the short name of the machine type - // resource, for example, `n1-standard-2`. - string machine_type_uri = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Disk option config settings. - DiskConfig disk_config = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Output only. Specifies that this instance group contains preemptible - // instances. - bool is_preemptible = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. Specifies the preemptibility of the instance group. - // - // The default value for master and worker groups is - // `NON_PREEMPTIBLE`. This default cannot be changed. - // - // The default value for secondary instances is - // `PREEMPTIBLE`. - Preemptibility preemptibility = 10 [(google.api.field_behavior) = OPTIONAL]; - - // Output only. The config for Compute Engine Instance Group - // Manager that manages this group. - // This is only used for preemptible instance groups. - ManagedGroupConfig managed_group_config = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. The Compute Engine accelerator configuration for these - // instances. - repeated AcceleratorConfig accelerators = 8 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Specifies the minimum cpu platform for the Instance Group. - // See [Dataproc -> Minimum CPU - // Platform](https://cloud.google.com/dataproc/docs/concepts/compute/dataproc-min-cpu). - string min_cpu_platform = 9 [(google.api.field_behavior) = OPTIONAL]; -} - -// Specifies the resources used to actively manage an instance group. -message ManagedGroupConfig { - // Output only. The name of the Instance Template used for the Managed - // Instance Group. - string instance_template_name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The name of the Instance Group Manager for this group. - string instance_group_manager_name = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Specifies the type and number of accelerator cards attached to the instances -// of an instance. See [GPUs on Compute -// Engine](https://cloud.google.com/compute/docs/gpus/). -message AcceleratorConfig { - // Full URL, partial URI, or short name of the accelerator type resource to - // expose to this instance. See - // [Compute Engine - // AcceleratorTypes](https://cloud.google.com/compute/docs/reference/beta/acceleratorTypes). - // - // Examples: - // - // * `https://www.googleapis.com/compute/beta/projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80` - // * `projects/[project_id]/zones/us-east1-a/acceleratorTypes/nvidia-tesla-k80` - // * `nvidia-tesla-k80` - // - // **Auto Zone Exception**: If you are using the Dataproc - // [Auto Zone - // Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement) - // feature, you must use the short name of the accelerator type - // resource, for example, `nvidia-tesla-k80`. - string accelerator_type_uri = 1; - - // The number of the accelerator cards of this type exposed to this instance. - int32 accelerator_count = 2; -} - -// Specifies the config of disk options for a group of VM instances. -message DiskConfig { - // Optional. Type of the boot disk (default is "pd-standard"). - // Valid values: "pd-balanced" (Persistent Disk Balanced Solid State Drive), - // "pd-ssd" (Persistent Disk Solid State Drive), - // or "pd-standard" (Persistent Disk Hard Disk Drive). - // See [Disk types](https://cloud.google.com/compute/docs/disks#disk-types). - string boot_disk_type = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Size in GB of the boot disk (default is 500GB). - int32 boot_disk_size_gb = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Number of attached SSDs, from 0 to 4 (default is 0). - // If SSDs are not attached, the boot disk is used to store runtime logs and - // [HDFS](https://hadoop.apache.org/docs/r1.2.1/hdfs_user_guide.html) data. - // If one or more SSDs are attached, this runtime bulk - // data is spread across them, and the boot disk contains only basic - // config and installed binaries. - int32 num_local_ssds = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Interface type of local SSDs (default is "scsi"). - // Valid values: "scsi" (Small Computer System Interface), - // "nvme" (Non-Volatile Memory Express). - // See [local SSD - // performance](https://cloud.google.com/compute/docs/disks/local-ssd#performance). - string local_ssd_interface = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// Specifies an executable to run on a fully configured node and a -// timeout period for executable completion. -message NodeInitializationAction { - // Required. Cloud Storage URI of executable file. - string executable_file = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Amount of time executable has to complete. Default is - // 10 minutes (see JSON representation of - // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). - // - // Cluster creation fails with an explanatory error message (the - // name of the executable that caused the error and the exceeded timeout - // period) if the executable is not completed at end of the timeout period. - google.protobuf.Duration execution_timeout = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// The status of a cluster and its instances. -message ClusterStatus { - // The cluster state. - enum State { - // The cluster state is unknown. - UNKNOWN = 0; - - // The cluster is being created and set up. It is not ready for use. - CREATING = 1; - - // The cluster is currently running and healthy. It is ready for use. - // - // **Note:** The cluster state changes from "creating" to "running" status - // after the master node(s), first two primary worker nodes (and the last - // primary worker node if primary workers > 2) are running. - RUNNING = 2; - - // The cluster encountered an error. It is not ready for use. - ERROR = 3; - - // The cluster has encountered an error while being updated. Jobs can - // be submitted to the cluster, but the cluster cannot be updated. - ERROR_DUE_TO_UPDATE = 9; - - // The cluster is being deleted. It cannot be used. - DELETING = 4; - - // The cluster is being updated. It continues to accept and process jobs. - UPDATING = 5; - - // The cluster is being stopped. It cannot be used. - STOPPING = 6; - - // The cluster is currently stopped. It is not ready for use. - STOPPED = 7; - - // The cluster is being started. It is not ready for use. - STARTING = 8; - } - - // The cluster substate. - enum Substate { - // The cluster substate is unknown. - UNSPECIFIED = 0; - - // The cluster is known to be in an unhealthy state - // (for example, critical daemons are not running or HDFS capacity is - // exhausted). - // - // Applies to RUNNING state. - UNHEALTHY = 1; - - // The agent-reported status is out of date (may occur if - // Dataproc loses communication with Agent). - // - // Applies to RUNNING state. - STALE_STATUS = 2; - } - - // Output only. The cluster's state. - State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. Output only. Details of cluster's state. - string detail = 2 [ - (google.api.field_behavior) = OUTPUT_ONLY, - (google.api.field_behavior) = OPTIONAL - ]; - - // Output only. Time when this state was entered (see JSON representation of - // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)). - google.protobuf.Timestamp state_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Additional state information that includes - // status reported by the agent. - Substate substate = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Security related configuration, including encryption, Kerberos, etc. -message SecurityConfig { - // Optional. Kerberos related configuration. - KerberosConfig kerberos_config = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Identity related configuration, including service account based - // secure multi-tenancy user mappings. - IdentityConfig identity_config = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// Specifies Kerberos related configuration. -message KerberosConfig { - // Optional. Flag to indicate whether to Kerberize the cluster (default: false). Set - // this field to true to enable Kerberos on a cluster. - bool enable_kerberos = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud Storage URI of a KMS encrypted file containing the root - // principal password. - string root_principal_password_uri = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The uri of the KMS key used to encrypt various sensitive - // files. - string kms_key_uri = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud Storage URI of the keystore file used for SSL - // encryption. If not provided, Dataproc will provide a self-signed - // certificate. - string keystore_uri = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud Storage URI of the truststore file used for SSL - // encryption. If not provided, Dataproc will provide a self-signed - // certificate. - string truststore_uri = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud Storage URI of a KMS encrypted file containing the - // password to the user provided keystore. For the self-signed certificate, - // this password is generated by Dataproc. - string keystore_password_uri = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud Storage URI of a KMS encrypted file containing the - // password to the user provided key. For the self-signed certificate, this - // password is generated by Dataproc. - string key_password_uri = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud Storage URI of a KMS encrypted file containing the - // password to the user provided truststore. For the self-signed certificate, - // this password is generated by Dataproc. - string truststore_password_uri = 8 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The remote realm the Dataproc on-cluster KDC will trust, should - // the user enable cross realm trust. - string cross_realm_trust_realm = 9 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The KDC (IP or hostname) for the remote trusted realm in a cross - // realm trust relationship. - string cross_realm_trust_kdc = 10 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The admin server (IP or hostname) for the remote trusted realm in - // a cross realm trust relationship. - string cross_realm_trust_admin_server = 11 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud Storage URI of a KMS encrypted file containing the - // shared password between the on-cluster Kerberos realm and the remote - // trusted realm, in a cross realm trust relationship. - string cross_realm_trust_shared_password_uri = 12 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud Storage URI of a KMS encrypted file containing the - // master key of the KDC database. - string kdc_db_key_uri = 13 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The lifetime of the ticket granting ticket, in hours. - // If not specified, or user specifies 0, then default value 10 - // will be used. - int32 tgt_lifetime_hours = 14 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The name of the on-cluster Kerberos realm. - // If not specified, the uppercased domain of hostnames will be the realm. - string realm = 15 [(google.api.field_behavior) = OPTIONAL]; -} - -// Identity related configuration, including service account based -// secure multi-tenancy user mappings. -message IdentityConfig { - // Required. Map of user to service account. - map<string, string> user_service_account_mapping = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// Specifies the selection and config of software inside the cluster. -message SoftwareConfig { - // Optional. The version of software inside the cluster. It must be one of the - // supported [Dataproc - // Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_dataproc_versions), - // such as "1.2" (including a subminor version, such as "1.2.29"), or the - // ["preview" - // version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). - // If unspecified, it defaults to the latest Debian version. - string image_version = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The properties to set on daemon config files. - // - // Property keys are specified in `prefix:property` format, for example - // `core:hadoop.tmp.dir`. The following are supported prefixes - // and their mappings: - // - // * capacity-scheduler: `capacity-scheduler.xml` - // * core: `core-site.xml` - // * distcp: `distcp-default.xml` - // * hdfs: `hdfs-site.xml` - // * hive: `hive-site.xml` - // * mapred: `mapred-site.xml` - // * pig: `pig.properties` - // * spark: `spark-defaults.conf` - // * yarn: `yarn-site.xml` - // - // For more information, see [Cluster - // properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties). - map<string, string> properties = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The set of components to activate on the cluster. - repeated Component optional_components = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// Specifies the cluster auto-delete schedule configuration. -message LifecycleConfig { - // Optional. The duration to keep the cluster alive while idling (when no jobs - // are running). Passing this threshold will cause the cluster to be - // deleted. Minimum value is 5 minutes; maximum value is 14 days (see JSON - // representation of - // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). - google.protobuf.Duration idle_delete_ttl = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Either the exact time the cluster should be deleted at or - // the cluster maximum age. - oneof ttl { - // Optional. The time when cluster will be auto-deleted (see JSON representation of - // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)). - google.protobuf.Timestamp auto_delete_time = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The lifetime duration of cluster. The cluster will be - // auto-deleted at the end of this period. Minimum value is 10 minutes; - // maximum value is 14 days (see JSON representation of - // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). - google.protobuf.Duration auto_delete_ttl = 3 [(google.api.field_behavior) = OPTIONAL]; - } - - // Output only. The time when cluster became idle (most recent job finished) - // and became eligible for deletion due to idleness (see JSON representation - // of - // [Timestamp](https://developers.google.com/protocol-buffers/docs/proto3#json)). - google.protobuf.Timestamp idle_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Specifies a Metastore configuration. -message MetastoreConfig { - // Required. Resource name of an existing Dataproc Metastore service. - // - // Example: - // - // * `projects/[project_id]/locations/[dataproc_region]/services/[service-name]` - string dataproc_metastore_service = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "metastore.googleapis.com/Service" - } - ]; -} - - -// Dataproc metric config. -message DataprocMetricConfig { - // A source for the collection of Dataproc OSS metrics (see [available OSS - // metrics] - // (https://cloud.google.com//dataproc/docs/guides/monitoring#available_oss_metrics)). - enum MetricSource { - // Required unspecified metric source. - METRIC_SOURCE_UNSPECIFIED = 0; - - // Default monitoring agent metrics. If this source is enabled, - // Dataproc enables the monitoring agent in Compute Engine, - // and collects default monitoring agent metrics, which are published - // with an `agent.googleapis.com` prefix. - MONITORING_AGENT_DEFAULTS = 1; - - // HDFS metric source. - HDFS = 2; - - // Spark metric source. - SPARK = 3; - - // YARN metric source. - YARN = 4; - - // Spark History Server metric source. - SPARK_HISTORY_SERVER = 5; - - // Hiveserver2 metric source. - HIVESERVER2 = 6; - } - - // A Dataproc OSS metric. - message Metric { - // Required. Default metrics are collected unless `metricOverrides` are - // specified for the metric source (see [Available OSS metrics] - // (https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics) - // for more information). - MetricSource metric_source = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Specify one or more [available OSS metrics] - // (https://cloud.google.com/dataproc/docs/guides/monitoring#available_oss_metrics) - // to collect for the metric course (for the `SPARK` metric source, any - // [Spark metric] - // (https://spark.apache.org/docs/latest/monitoring.html#metrics) can be - // specified). - // - // Provide metrics in the following format: - // <code><var>METRIC_SOURCE</var>:<var>INSTANCE</var>:<var>GROUP</var>:<var>METRIC</var></code> - // Use camelcase as appropriate. - // - // Examples: - // - // ``` - // yarn:ResourceManager:QueueMetrics:AppsCompleted - // spark:driver:DAGScheduler:job.allJobs - // sparkHistoryServer:JVM:Memory:NonHeapMemoryUsage.committed - // hiveserver2:JVM:Memory:NonHeapMemoryUsage.used - // ``` - // - // Notes: - // - // * Only the specified overridden metrics will be collected for the - // metric source. For example, if one or more `spark:executive` metrics - // are listed as metric overrides, other `SPARK` metrics will not be - // collected. The collection of the default metrics for other OSS metric - // sources is unaffected. For example, if both `SPARK` andd `YARN` metric - // sources are enabled, and overrides are provided for Spark metrics only, - // all default YARN metrics will be collected. - repeated string metric_overrides = 2 [(google.api.field_behavior) = OPTIONAL]; - } - - // Required. Metrics sources to enable. - repeated Metric metrics = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// Contains cluster daemon metrics, such as HDFS and YARN stats. -// -// **Beta Feature**: This report is available for testing purposes only. It may -// be changed before final release. -message ClusterMetrics { - // The HDFS metrics. - map<string, int64> hdfs_metrics = 1; - - // The YARN metrics. - map<string, int64> yarn_metrics = 2; -} - -// A request to create a cluster. -message CreateClusterRequest { - // Required. The ID of the Google Cloud Platform project that the cluster - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster to create. - Cluster cluster = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. A unique ID used to identify the request. If the server receives two - // [CreateClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.CreateClusterRequest)s - // with the same id, then the second request will be ignored and the - // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the backend - // is returned. - // - // It is recommended to always set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The ID must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Failure action when primary worker creation fails. - FailureAction action_on_failed_primary_workers = 5 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to update a cluster. -message UpdateClusterRequest { - // Required. The ID of the Google Cloud Platform project the - // cluster belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 5 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster name. - string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The changes to the cluster. - Cluster cluster = 3 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Timeout for graceful YARN decomissioning. Graceful - // decommissioning allows removing nodes from the cluster without - // interrupting jobs in progress. Timeout specifies how long to wait for jobs - // in progress to finish before forcefully removing nodes (and potentially - // interrupting jobs). Default timeout is 0 (for forceful decommission), and - // the maximum allowed timeout is 1 day. (see JSON representation of - // [Duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). - // - // Only supported on Dataproc image versions 1.2 and higher. - google.protobuf.Duration graceful_decommission_timeout = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Required. Specifies the path, relative to `Cluster`, of - // the field to update. For example, to change the number of workers - // in a cluster to 5, the `update_mask` parameter would be - // specified as `config.worker_config.num_instances`, - // and the `PATCH` request body would specify the new value, as follows: - // - // { - // "config":{ - // "workerConfig":{ - // "numInstances":"5" - // } - // } - // } - // Similarly, to change the number of preemptible workers in a cluster to 5, - // the `update_mask` parameter would be - // `config.secondary_worker_config.num_instances`, and the `PATCH` request - // body would be set as follows: - // - // { - // "config":{ - // "secondaryWorkerConfig":{ - // "numInstances":"5" - // } - // } - // } - // <strong>Note:</strong> Currently, only the following fields can be updated: - // - // <table> - // <tbody> - // <tr> - // <td><strong>Mask</strong></td> - // <td><strong>Purpose</strong></td> - // </tr> - // <tr> - // <td><strong><em>labels</em></strong></td> - // <td>Update labels</td> - // </tr> - // <tr> - // <td><strong><em>config.worker_config.num_instances</em></strong></td> - // <td>Resize primary worker group</td> - // </tr> - // <tr> - // <td><strong><em>config.secondary_worker_config.num_instances</em></strong></td> - // <td>Resize secondary worker group</td> - // </tr> - // <tr> - // <td>config.autoscaling_config.policy_uri</td><td>Use, stop using, or - // change autoscaling policies</td> - // </tr> - // </tbody> - // </table> - google.protobuf.FieldMask update_mask = 4 [(google.api.field_behavior) = REQUIRED]; - - // Optional. A unique ID used to identify the request. If the server - // receives two - // [UpdateClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.UpdateClusterRequest)s - // with the same id, then the second request will be ignored and the - // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the - // backend is returned. - // - // It is recommended to always set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The ID must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to stop a cluster. -message StopClusterRequest { - // Required. The ID of the Google Cloud Platform project the - // cluster belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster name. - string cluster_name = 3 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Specifying the `cluster_uuid` means the RPC will fail - // (with error NOT_FOUND) if a cluster with the specified UUID does not exist. - string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A unique ID used to identify the request. If the server - // receives two - // [StopClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.StopClusterRequest)s - // with the same id, then the second request will be ignored and the - // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the - // backend is returned. - // - // Recommendation: Set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The ID must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to start a cluster. -message StartClusterRequest { - // Required. The ID of the Google Cloud Platform project the - // cluster belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster name. - string cluster_name = 3 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Specifying the `cluster_uuid` means the RPC will fail - // (with error NOT_FOUND) if a cluster with the specified UUID does not exist. - string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A unique ID used to identify the request. If the server - // receives two - // [StartClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.StartClusterRequest)s - // with the same id, then the second request will be ignored and the - // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the - // backend is returned. - // - // Recommendation: Set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The ID must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to delete a cluster. -message DeleteClusterRequest { - // Required. The ID of the Google Cloud Platform project that the cluster - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster name. - string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Specifying the `cluster_uuid` means the RPC should fail - // (with error NOT_FOUND) if cluster with specified UUID does not exist. - string cluster_uuid = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A unique ID used to identify the request. If the server - // receives two - // [DeleteClusterRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.DeleteClusterRequest)s - // with the same id, then the second request will be ignored and the - // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the - // backend is returned. - // - // It is recommended to always set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The ID must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; -} - -// Request to get the resource representation for a cluster in a project. -message GetClusterRequest { - // Required. The ID of the Google Cloud Platform project that the cluster - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster name. - string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to list the clusters in a project. -message ListClustersRequest { - // Required. The ID of the Google Cloud Platform project that the cluster - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 4 [(google.api.field_behavior) = REQUIRED]; - - // Optional. A filter constraining the clusters to list. Filters are - // case-sensitive and have the following syntax: - // - // field = value [AND [field = value]] ... - // - // where **field** is one of `status.state`, `clusterName`, or `labels.[KEY]`, - // and `[KEY]` is a label key. **value** can be `*` to match all values. - // `status.state` can be one of the following: `ACTIVE`, `INACTIVE`, - // `CREATING`, `RUNNING`, `ERROR`, `DELETING`, or `UPDATING`. `ACTIVE` - // contains the `CREATING`, `UPDATING`, and `RUNNING` states. `INACTIVE` - // contains the `DELETING` and `ERROR` states. - // `clusterName` is the name of the cluster provided at creation time. - // Only the logical `AND` operator is supported; space-separated items are - // treated as having an implicit `AND` operator. - // - // Example filter: - // - // status.state = ACTIVE AND clusterName = mycluster - // AND labels.env = staging AND labels.starred = * - string filter = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The standard List page size. - int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The standard List page token. - string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// The list of all clusters in a project. -message ListClustersResponse { - // Output only. The clusters in the project. - repeated Cluster clusters = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. This token is included in the response if there are more - // results to fetch. To fetch additional results, provide this value as the - // `page_token` in a subsequent `ListClustersRequest`. - string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// A request to collect cluster diagnostic information. -message DiagnoseClusterRequest { - // Required. The ID of the Google Cloud Platform project that the cluster - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster name. - string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// The location of diagnostic output. -message DiagnoseClusterResults { - // Output only. The Cloud Storage URI of the diagnostic output. - // The output report is a plain text file with a summary of collected - // diagnostics. - string output_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Reservation Affinity for consuming Zonal reservation. -message ReservationAffinity { - // Indicates whether to consume capacity from an reservation or not. - enum Type { - TYPE_UNSPECIFIED = 0; - - // Do not consume from any allocated capacity. - NO_RESERVATION = 1; - - // Consume any reservation available. - ANY_RESERVATION = 2; - - // Must consume from a specific reservation. Must specify key value fields - // for specifying the reservations. - SPECIFIC_RESERVATION = 3; - } - - // Optional. Type of reservation to consume - Type consume_reservation_type = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Corresponds to the label key of reservation resource. - string key = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Corresponds to the label values of reservation resource. - repeated string values = 3 [(google.api.field_behavior) = OPTIONAL]; -} diff --git a/third_party/googleapis/google/cloud/dataproc/v1/dataproc_gapic.yaml b/third_party/googleapis/google/cloud/dataproc/v1/dataproc_gapic.yaml deleted file mode 100644 index bc52ee4..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/dataproc_gapic.yaml +++ /dev/null @@ -1,43 +0,0 @@ -type: com.google.api.codegen.ConfigProto -config_schema_version: 2.0.0 -interfaces: -- name: google.cloud.dataproc.v1.ClusterController - methods: - - name: CreateCluster - long_running: - initial_poll_delay_millis: 1000 - poll_delay_multiplier: 2 - max_poll_delay_millis: 10000 - total_poll_timeout_millis: 900000 - - name: UpdateCluster - long_running: - initial_poll_delay_millis: 1000 - poll_delay_multiplier: 2 - max_poll_delay_millis: 10000 - total_poll_timeout_millis: 900000 - - name: DeleteCluster - long_running: - initial_poll_delay_millis: 1000 - poll_delay_multiplier: 2 - max_poll_delay_millis: 10000 - total_poll_timeout_millis: 900000 - - name: DiagnoseCluster - long_running: - initial_poll_delay_millis: 1000 - poll_delay_multiplier: 2 - max_poll_delay_millis: 10000 - total_poll_timeout_millis: 30000 -- name: google.cloud.dataproc.v1.WorkflowTemplateService - methods: - - name: InstantiateWorkflowTemplate - long_running: - initial_poll_delay_millis: 1000 - poll_delay_multiplier: 2 - max_poll_delay_millis: 10000 - total_poll_timeout_millis: 43200000 - - name: InstantiateInlineWorkflowTemplate - long_running: - initial_poll_delay_millis: 1000 - poll_delay_multiplier: 2 - max_poll_delay_millis: 10000 - total_poll_timeout_millis: 43200000 diff --git a/third_party/googleapis/google/cloud/dataproc/v1/dataproc_grpc_service_config.json b/third_party/googleapis/google/cloud/dataproc/v1/dataproc_grpc_service_config.json deleted file mode 100755 index 411b7ba..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/dataproc_grpc_service_config.json +++ /dev/null @@ -1,207 +0,0 @@ -{ - "methodConfig": [ - { - "name": [ - { - "service": "google.cloud.dataproc.v1.AutoscalingPolicyService", - "method": "CreateAutoscalingPolicy" - }, - { - "service": "google.cloud.dataproc.v1.AutoscalingPolicyService", - "method": "DeleteAutoscalingPolicy" - } - ], - "timeout": "600s" - }, - { - "name": [ - { - "service": "google.cloud.dataproc.v1.AutoscalingPolicyService", - "method": "UpdateAutoscalingPolicy" - }, - { - "service": "google.cloud.dataproc.v1.AutoscalingPolicyService", - "method": "GetAutoscalingPolicy" - }, - { - "service": "google.cloud.dataproc.v1.AutoscalingPolicyService", - "method": "ListAutoscalingPolicies" - } - ], - "timeout": "600s", - "retryPolicy": { - "initialBackoff": "0.100s", - "maxBackoff": "60s", - "backoffMultiplier": 1.3, - "retryableStatusCodes": [ - "DEADLINE_EXCEEDED", - "UNAVAILABLE" - ] - } - }, - { - "name": [ - { - "service": "google.cloud.dataproc.v1.JobController", - "method": "GetJob" - }, - { - "service": "google.cloud.dataproc.v1.JobController", - "method": "ListJobs" - }, - { - "service": "google.cloud.dataproc.v1.JobController", - "method": "CancelJob" - } - ], - "timeout": "900s", - "retryPolicy": { - "initialBackoff": "0.100s", - "maxBackoff": "60s", - "backoffMultiplier": 1.3, - "retryableStatusCodes": [ - "DEADLINE_EXCEEDED", - "INTERNAL", - "UNAVAILABLE" - ] - } - }, - { - "name": [ - { - "service": "google.cloud.dataproc.v1.WorkflowTemplateService", - "method": "CreateWorkflowTemplate" - }, - { - "service": "google.cloud.dataproc.v1.WorkflowTemplateService", - "method": "InstantiateWorkflowTemplate" - }, - { - "service": "google.cloud.dataproc.v1.WorkflowTemplateService", - "method": "InstantiateInlineWorkflowTemplate" - }, - { - "service": "google.cloud.dataproc.v1.WorkflowTemplateService", - "method": "UpdateWorkflowTemplate" - }, - { - "service": "google.cloud.dataproc.v1.WorkflowTemplateService", - "method": "DeleteWorkflowTemplate" - } - ], - "timeout": "600s", - "retryPolicy": { - "initialBackoff": "0.100s", - "maxBackoff": "60s", - "backoffMultiplier": 1.3, - "retryableStatusCodes": [ - "UNAVAILABLE" - ] - } - }, - { - "name": [ - { - "service": "google.cloud.dataproc.v1.WorkflowTemplateService", - "method": "GetWorkflowTemplate" - }, - { - "service": "google.cloud.dataproc.v1.WorkflowTemplateService", - "method": "ListWorkflowTemplates" - } - ], - "timeout": "600s", - "retryPolicy": { - "initialBackoff": "0.100s", - "maxBackoff": "60s", - "backoffMultiplier": 1.3, - "retryableStatusCodes": [ - "DEADLINE_EXCEEDED", - "INTERNAL", - "UNAVAILABLE" - ] - } - }, - { - "name": [ - { - "service": "google.cloud.dataproc.v1.ClusterController", - "method": "CreateCluster" - }, - { - "service": "google.cloud.dataproc.v1.ClusterController", - "method": "UpdateCluster" - }, - { - "service": "google.cloud.dataproc.v1.ClusterController", - "method": "DeleteCluster" - }, - { - "service": "google.cloud.dataproc.v1.ClusterController", - "method": "DiagnoseCluster" - } - ], - "timeout": "300s", - "retryPolicy": { - "initialBackoff": "0.100s", - "maxBackoff": "60s", - "backoffMultiplier": 1.3, - "retryableStatusCodes": [ - "UNAVAILABLE" - ] - } - }, - { - "name": [ - { - "service": "google.cloud.dataproc.v1.ClusterController", - "method": "GetCluster" - }, - { - "service": "google.cloud.dataproc.v1.ClusterController", - "method": "ListClusters" - } - ], - "timeout": "300s", - "retryPolicy": { - "initialBackoff": "0.100s", - "maxBackoff": "60s", - "backoffMultiplier": 1.3, - "retryableStatusCodes": [ - "INTERNAL", - "DEADLINE_EXCEEDED", - "UNAVAILABLE" - ] - } - }, - { - "name": [ - { - "service": "google.cloud.dataproc.v1.JobController", - "method": "SubmitJob" - }, - { - "service": "google.cloud.dataproc.v1.JobController", - "method": "SubmitJobAsOperation" - }, - { - "service": "google.cloud.dataproc.v1.JobController", - "method": "UpdateJob" - }, - { - "service": "google.cloud.dataproc.v1.JobController", - "method": "DeleteJob" - } - ], - "timeout": "900s", - "retryPolicy": { - "initialBackoff": "0.100s", - "maxBackoff": "60s", - "backoffMultiplier": 1.3, - "retryableStatusCodes": [ - "UNAVAILABLE" - ] - } - } - ] -} diff --git a/third_party/googleapis/google/cloud/dataproc/v1/dataproc_v1.yaml b/third_party/googleapis/google/cloud/dataproc/v1/dataproc_v1.yaml deleted file mode 100644 index b476270..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/dataproc_v1.yaml +++ /dev/null @@ -1,134 +0,0 @@ -type: google.api.Service -config_version: 3 -name: dataproc.googleapis.com -title: Cloud Dataproc API - -apis: -- name: google.cloud.dataproc.v1.AutoscalingPolicyService -- name: google.cloud.dataproc.v1.BatchController -- name: google.cloud.dataproc.v1.ClusterController -- name: google.cloud.dataproc.v1.JobController -- name: google.cloud.dataproc.v1.WorkflowTemplateService - -types: -- name: google.cloud.dataproc.v1.BatchOperationMetadata -- name: google.cloud.dataproc.v1.ClusterOperationMetadata -- name: google.cloud.dataproc.v1.DiagnoseClusterResults -- name: google.cloud.dataproc.v1.JobMetadata -- name: google.cloud.dataproc.v1.WorkflowMetadata - -documentation: - summary: 'Manages Hadoop-based clusters and jobs on Google Cloud Platform.' - rules: - - selector: google.iam.v1.IAMPolicy.GetIamPolicy - description: |- - Gets the access control policy for a resource. Returns an empty policy - if the resource exists and does not have a policy set. - - - selector: google.iam.v1.IAMPolicy.SetIamPolicy - description: |- - Sets the access control policy on the specified resource. Replaces - any existing policy. - - Can return `NOT_FOUND`, `INVALID_ARGUMENT`, and `PERMISSION_DENIED` - errors. - - - selector: google.iam.v1.IAMPolicy.TestIamPermissions - description: |- - Returns permissions that a caller has on the specified resource. If the - resource does not exist, this will return an empty set of - permissions, not a `NOT_FOUND` error. - - Note: This operation is designed to be used for building - permission-aware UIs and command-line tools, not for authorization - checking. This operation may "fail open" without warning. - -http: - rules: - - selector: google.iam.v1.IAMPolicy.GetIamPolicy - post: '/v1/{resource=projects/*/regions/*/clusters/*}:getIamPolicy' - body: '*' - additional_bindings: - - post: '/v1/{resource=projects/*/regions/*/jobs/*}:getIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/operations/*}:getIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:getIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:getIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:getIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:getIamPolicy' - body: '*' - - selector: google.iam.v1.IAMPolicy.SetIamPolicy - post: '/v1/{resource=projects/*/regions/*/clusters/*}:setIamPolicy' - body: '*' - additional_bindings: - - post: '/v1/{resource=projects/*/regions/*/jobs/*}:setIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/operations/*}:setIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:setIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:setIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:setIamPolicy' - body: '*' - - post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:setIamPolicy' - body: '*' - - selector: google.iam.v1.IAMPolicy.TestIamPermissions - post: '/v1/{resource=projects/*/regions/*/clusters/*}:testIamPermissions' - body: '*' - additional_bindings: - - post: '/v1/{resource=projects/*/regions/*/jobs/*}:testIamPermissions' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/operations/*}:testIamPermissions' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/workflowTemplates/*}:testIamPermissions' - body: '*' - - post: '/v1/{resource=projects/*/locations/*/workflowTemplates/*}:testIamPermissions' - body: '*' - - post: '/v1/{resource=projects/*/regions/*/autoscalingPolicies/*}:testIamPermissions' - body: '*' - - post: '/v1/{resource=projects/*/locations/*/autoscalingPolicies/*}:testIamPermissions' - body: '*' - - selector: google.longrunning.Operations.CancelOperation - post: '/v1/{name=projects/*/regions/*/operations/*}:cancel' - - selector: google.longrunning.Operations.DeleteOperation - delete: '/v1/{name=projects/*/regions/*/operations/*}' - - selector: google.longrunning.Operations.GetOperation - get: '/v1/{name=projects/*/regions/*/operations/*}' - - selector: google.longrunning.Operations.ListOperations - get: '/v1/{name=projects/*/regions/*/operations}' - -authentication: - rules: - - selector: 'google.cloud.dataproc.v1.AutoscalingPolicyService.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.cloud.dataproc.v1.BatchController.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.cloud.dataproc.v1.ClusterController.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.cloud.dataproc.v1.JobController.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.cloud.dataproc.v1.WorkflowTemplateService.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.iam.v1.IAMPolicy.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.longrunning.Operations.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform diff --git a/third_party/googleapis/google/cloud/dataproc/v1/jobs.proto b/third_party/googleapis/google/cloud/dataproc/v1/jobs.proto deleted file mode 100644 index 823a12c..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/jobs.proto +++ /dev/null @@ -1,924 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.v1; - -import "google/api/annotations.proto"; -import "google/api/client.proto"; -import "google/api/field_behavior.proto"; -import "google/longrunning/operations.proto"; -import "google/protobuf/empty.proto"; -import "google/protobuf/field_mask.proto"; -import "google/protobuf/timestamp.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; -option java_multiple_files = true; -option java_outer_classname = "JobsProto"; -option java_package = "com.google.cloud.dataproc.v1"; - -// The JobController provides methods to manage jobs. -service JobController { - option (google.api.default_host) = "dataproc.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; - - // Submits a job to a cluster. - rpc SubmitJob(SubmitJobRequest) returns (Job) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/jobs:submit" - body: "*" - }; - option (google.api.method_signature) = "project_id,region,job"; - } - - // Submits job to a cluster. - rpc SubmitJobAsOperation(SubmitJobRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/jobs:submitAsOperation" - body: "*" - }; - option (google.api.method_signature) = "project_id, region, job"; - option (google.longrunning.operation_info) = { - response_type: "Job" - metadata_type: "JobMetadata" - }; - } - - // Gets the resource representation for a job in a project. - rpc GetJob(GetJobRequest) returns (Job) { - option (google.api.http) = { - get: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" - }; - option (google.api.method_signature) = "project_id,region,job_id"; - } - - // Lists regions/{region}/jobs in a project. - rpc ListJobs(ListJobsRequest) returns (ListJobsResponse) { - option (google.api.http) = { - get: "/v1/projects/{project_id}/regions/{region}/jobs" - }; - option (google.api.method_signature) = "project_id,region"; - option (google.api.method_signature) = "project_id,region,filter"; - } - - // Updates a job in a project. - rpc UpdateJob(UpdateJobRequest) returns (Job) { - option (google.api.http) = { - patch: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" - body: "job" - }; - } - - // Starts a job cancellation request. To access the job resource - // after cancellation, call - // [regions/{region}/jobs.list](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs/list) - // or - // [regions/{region}/jobs.get](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs/get). - rpc CancelJob(CancelJobRequest) returns (Job) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}:cancel" - body: "*" - }; - option (google.api.method_signature) = "project_id,region,job_id"; - } - - // Deletes the job from the project. If the job is active, the delete fails, - // and the response returns `FAILED_PRECONDITION`. - rpc DeleteJob(DeleteJobRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { - delete: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" - }; - option (google.api.method_signature) = "project_id,region,job_id"; - } -} - -// The runtime logging config of the job. -message LoggingConfig { - // The Log4j level for job execution. When running an - // [Apache Hive](https://hive.apache.org/) job, Cloud - // Dataproc configures the Hive client to an equivalent verbosity level. - enum Level { - // Level is unspecified. Use default level for log4j. - LEVEL_UNSPECIFIED = 0; - - // Use ALL level for log4j. - ALL = 1; - - // Use TRACE level for log4j. - TRACE = 2; - - // Use DEBUG level for log4j. - DEBUG = 3; - - // Use INFO level for log4j. - INFO = 4; - - // Use WARN level for log4j. - WARN = 5; - - // Use ERROR level for log4j. - ERROR = 6; - - // Use FATAL level for log4j. - FATAL = 7; - - // Turn off log4j. - OFF = 8; - } - - // The per-package log levels for the driver. This may include - // "root" package name to configure rootLogger. - // Examples: - // 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG' - map<string, Level> driver_log_levels = 2; -} - -// A Dataproc job for running -// [Apache Hadoop -// MapReduce](https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduceTutorial.html) -// jobs on [Apache Hadoop -// YARN](https://hadoop.apache.org/docs/r2.7.1/hadoop-yarn/hadoop-yarn-site/YARN.html). -message HadoopJob { - // Required. Indicates the location of the driver's main class. Specify - // either the jar file that contains the main class or the main class name. - // To specify both, add the jar file to `jar_file_uris`, and then specify - // the main class name in this property. - oneof driver { - // The HCFS URI of the jar file containing the main class. - // Examples: - // 'gs://foo-bucket/analytics-binaries/extract-useful-metrics-mr.jar' - // 'hdfs:/tmp/test-samples/custom-wordcount.jar' - // 'file:///home/usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar' - string main_jar_file_uri = 1; - - // The name of the driver's main class. The jar file containing the class - // must be in the default CLASSPATH or specified in `jar_file_uris`. - string main_class = 2; - } - - // Optional. The arguments to pass to the driver. Do not - // include arguments, such as `-libjars` or `-Dfoo=bar`, that can be set as - // job properties, since a collision may occur that causes an incorrect job - // submission. - repeated string args = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Jar file URIs to add to the CLASSPATHs of the - // Hadoop driver and tasks. - repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS (Hadoop Compatible Filesystem) URIs of files to be copied - // to the working directory of Hadoop drivers and distributed tasks. Useful - // for naively parallel tasks. - repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted in the working directory of - // Hadoop drivers and tasks. Supported file types: - // .jar, .tar, .tar.gz, .tgz, or .zip. - repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure Hadoop. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in /etc/hadoop/conf/*-site and - // classes in user code. - map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running [Apache Spark](http://spark.apache.org/) -// applications on YARN. -message SparkJob { - // Required. The specification of the main method to call to drive the job. - // Specify either the jar file that contains the main class or the main class - // name. To pass both a main jar and a main class in that jar, add the jar to - // `CommonJob.jar_file_uris`, and then specify the main class name in - // `main_class`. - oneof driver { - // The HCFS URI of the jar file that contains the main class. - string main_jar_file_uri = 1; - - // The name of the driver's main class. The jar file that contains the class - // must be in the default CLASSPATH or specified in `jar_file_uris`. - string main_class = 2; - } - - // Optional. The arguments to pass to the driver. Do not include arguments, - // such as `--conf`, that can be set as job properties, since a collision may - // occur that causes an incorrect job submission. - repeated string args = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the CLASSPATHs of the - // Spark driver and tasks. - repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. Useful for naively parallel tasks. - repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // .jar, .tar, .tar.gz, .tgz, and .zip. - repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure Spark. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in - // /etc/spark/conf/spark-defaults.conf and classes in user code. - map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running -// [Apache -// PySpark](https://spark.apache.org/docs/0.9.0/python-programming-guide.html) -// applications on YARN. -message PySparkJob { - // Required. The HCFS URI of the main Python file to use as the driver. Must - // be a .py file. - string main_python_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The arguments to pass to the driver. Do not include arguments, - // such as `--conf`, that can be set as job properties, since a collision may - // occur that causes an incorrect job submission. - repeated string args = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS file URIs of Python files to pass to the PySpark - // framework. Supported file types: .py, .egg, and .zip. - repeated string python_file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the CLASSPATHs of the - // Python driver and tasks. - repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. Useful for naively parallel tasks. - repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // .jar, .tar, .tar.gz, .tgz, and .zip. - repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure PySpark. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in - // /etc/spark/conf/spark-defaults.conf and classes in user code. - map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; -} - -// A list of queries to run on a cluster. -message QueryList { - // Required. The queries to execute. You do not need to end a query expression - // with a semicolon. Multiple queries can be specified in one - // string by separating each with a semicolon. Here is an example of a - // Dataproc API snippet that uses a QueryList to specify a HiveJob: - // - // "hiveJob": { - // "queryList": { - // "queries": [ - // "query1", - // "query2", - // "query3;query4", - // ] - // } - // } - repeated string queries = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// A Dataproc job for running [Apache Hive](https://hive.apache.org/) -// queries on YARN. -message HiveJob { - // Required. The sequence of Hive queries to execute, specified as either - // an HCFS file URI or a list of queries. - oneof queries { - // The HCFS URI of the script that contains Hive queries. - string query_file_uri = 1; - - // A list of queries. - QueryList query_list = 2; - } - - // Optional. Whether to continue executing queries if a query fails. - // The default value is `false`. Setting to `true` can be useful when - // executing independent parallel queries. - bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Mapping of query variable names to values (equivalent to the - // Hive command: `SET name="value";`). - map<string, string> script_variables = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names and values, used to configure Hive. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml, - // /etc/hive/conf/hive-site.xml, and classes in user code. - map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the CLASSPATH of the - // Hive server and Hadoop MapReduce (MR) tasks. Can contain Hive SerDes - // and UDFs. - repeated string jar_file_uris = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running [Apache Spark -// SQL](http://spark.apache.org/sql/) queries. -message SparkSqlJob { - // Required. The sequence of Spark SQL queries to execute, specified as - // either an HCFS file URI or as a list of queries. - oneof queries { - // The HCFS URI of the script that contains SQL queries. - string query_file_uri = 1; - - // A list of queries. - QueryList query_list = 2; - } - - // Optional. Mapping of query variable names to values (equivalent to the - // Spark SQL command: SET `name="value";`). - map<string, string> script_variables = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure - // Spark SQL's SparkConf. Properties that conflict with values set by the - // Dataproc API may be overwritten. - map<string, string> properties = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH. - repeated string jar_file_uris = 56 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running [Apache Pig](https://pig.apache.org/) -// queries on YARN. -message PigJob { - // Required. The sequence of Pig queries to execute, specified as an HCFS - // file URI or a list of queries. - oneof queries { - // The HCFS URI of the script that contains the Pig queries. - string query_file_uri = 1; - - // A list of queries. - QueryList query_list = 2; - } - - // Optional. Whether to continue executing queries if a query fails. - // The default value is `false`. Setting to `true` can be useful when - // executing independent parallel queries. - bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Mapping of query variable names to values (equivalent to the Pig - // command: `name=[value]`). - map<string, string> script_variables = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure Pig. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml, - // /etc/pig/conf/pig.properties, and classes in user code. - map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the CLASSPATH of - // the Pig Client and Hadoop MapReduce (MR) tasks. Can contain Pig UDFs. - repeated string jar_file_uris = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running -// [Apache SparkR](https://spark.apache.org/docs/latest/sparkr.html) -// applications on YARN. -message SparkRJob { - // Required. The HCFS URI of the main R file to use as the driver. - // Must be a .R file. - string main_r_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The arguments to pass to the driver. Do not include arguments, - // such as `--conf`, that can be set as job properties, since a collision may - // occur that causes an incorrect job submission. - repeated string args = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. Useful for naively parallel tasks. - repeated string file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // .jar, .tar, .tar.gz, .tgz, and .zip. - repeated string archive_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure SparkR. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in - // /etc/spark/conf/spark-defaults.conf and classes in user code. - map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running [Presto](https://prestosql.io/) queries. -// **IMPORTANT**: The [Dataproc Presto Optional -// Component](https://cloud.google.com/dataproc/docs/concepts/components/presto) -// must be enabled when the cluster is created to submit a Presto job to the -// cluster. -message PrestoJob { - // Required. The sequence of Presto queries to execute, specified as - // either an HCFS file URI or as a list of queries. - oneof queries { - // The HCFS URI of the script that contains SQL queries. - string query_file_uri = 1; - - // A list of queries. - QueryList query_list = 2; - } - - // Optional. Whether to continue executing queries if a query fails. - // The default value is `false`. Setting to `true` can be useful when - // executing independent parallel queries. - bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The format in which query output will be displayed. See the - // Presto documentation for supported output formats - string output_format = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Presto client tags to attach to this query - repeated string client_tags = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values. Used to set Presto - // [session properties](https://prestodb.io/docs/current/sql/set-session.html) - // Equivalent to using the --session flag in the Presto CLI - map<string, string> properties = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// Dataproc job config. -message JobPlacement { - // Required. The name of the cluster where the job will be submitted. - string cluster_name = 1 [(google.api.field_behavior) = REQUIRED]; - - // Output only. A cluster UUID generated by the Dataproc service when - // the job is submitted. - string cluster_uuid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. Cluster labels to identify a cluster where the job will be submitted. - map<string, string> cluster_labels = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// Dataproc job status. -message JobStatus { - // The job state. - enum State { - // The job state is unknown. - STATE_UNSPECIFIED = 0; - - // The job is pending; it has been submitted, but is not yet running. - PENDING = 1; - - // Job has been received by the service and completed initial setup; - // it will soon be submitted to the cluster. - SETUP_DONE = 8; - - // The job is running on the cluster. - RUNNING = 2; - - // A CancelJob request has been received, but is pending. - CANCEL_PENDING = 3; - - // Transient in-flight resources have been canceled, and the request to - // cancel the running job has been issued to the cluster. - CANCEL_STARTED = 7; - - // The job cancellation was successful. - CANCELLED = 4; - - // The job has completed successfully. - DONE = 5; - - // The job has completed, but encountered an error. - ERROR = 6; - - // Job attempt has failed. The detail field contains failure details for - // this attempt. - // - // Applies to restartable jobs only. - ATTEMPT_FAILURE = 9; - } - - // The job substate. - enum Substate { - // The job substate is unknown. - UNSPECIFIED = 0; - - // The Job is submitted to the agent. - // - // Applies to RUNNING state. - SUBMITTED = 1; - - // The Job has been received and is awaiting execution (it may be waiting - // for a condition to be met). See the "details" field for the reason for - // the delay. - // - // Applies to RUNNING state. - QUEUED = 2; - - // The agent-reported status is out of date, which may be caused by a - // loss of communication between the agent and Dataproc. If the - // agent does not send a timely update, the job will fail. - // - // Applies to RUNNING state. - STALE_STATUS = 3; - } - - // Output only. A state message specifying the overall job state. - State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. Output only. Job state details, such as an error - // description if the state is <code>ERROR</code>. - string details = 2 [ - (google.api.field_behavior) = OUTPUT_ONLY, - (google.api.field_behavior) = OPTIONAL - ]; - - // Output only. The time when this state was entered. - google.protobuf.Timestamp state_start_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Additional state information, which includes - // status reported by the agent. - Substate substate = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Encapsulates the full scoping used to reference a job. -message JobReference { - // Optional. The ID of the Google Cloud Platform project that the job belongs to. If - // specified, must match the request project ID. - string project_id = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The job ID, which must be unique within the project. - // - // The ID must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), or hyphens (-). The maximum length is 100 characters. - // - // If not specified by the caller, the job ID will be provided by the server. - string job_id = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// A YARN application created by a job. Application information is a subset of -// <code>org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto</code>. -// -// **Beta Feature**: This report is available for testing purposes only. It may -// be changed before final release. -message YarnApplication { - // The application state, corresponding to - // <code>YarnProtos.YarnApplicationStateProto</code>. - enum State { - // Status is unspecified. - STATE_UNSPECIFIED = 0; - - // Status is NEW. - NEW = 1; - - // Status is NEW_SAVING. - NEW_SAVING = 2; - - // Status is SUBMITTED. - SUBMITTED = 3; - - // Status is ACCEPTED. - ACCEPTED = 4; - - // Status is RUNNING. - RUNNING = 5; - - // Status is FINISHED. - FINISHED = 6; - - // Status is FAILED. - FAILED = 7; - - // Status is KILLED. - KILLED = 8; - } - - // Required. The application name. - string name = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The application state. - State state = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The numerical progress of the application, from 1 to 100. - float progress = 3 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The HTTP URL of the ApplicationMaster, HistoryServer, or - // TimelineServer that provides application-specific information. The URL uses - // the internal hostname, and requires a proxy server for resolution and, - // possibly, access. - string tracking_url = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job resource. -message Job { - // Optional. The fully qualified reference to the job, which can be used to - // obtain the equivalent REST path of the job resource. If this property - // is not specified when a job is created, the server generates a - // <code>job_id</code>. - JobReference reference = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Required. Job information, including how, when, and where to - // run the job. - JobPlacement placement = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The application/framework-specific portion of the job. - oneof type_job { - // Optional. Job is a Hadoop job. - HadoopJob hadoop_job = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Spark job. - SparkJob spark_job = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a PySpark job. - PySparkJob pyspark_job = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Hive job. - HiveJob hive_job = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Pig job. - PigJob pig_job = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a SparkR job. - SparkRJob spark_r_job = 21 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a SparkSql job. - SparkSqlJob spark_sql_job = 12 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Presto job. - PrestoJob presto_job = 23 [(google.api.field_behavior) = OPTIONAL]; - } - - // Output only. The job status. Additional application-specific - // status information may be contained in the <code>type_job</code> - // and <code>yarn_applications</code> fields. - JobStatus status = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The previous job status. - repeated JobStatus status_history = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The collection of YARN applications spun up by this job. - // - // **Beta** Feature: This report is available for testing purposes only. It - // may be changed before final release. - repeated YarnApplication yarn_applications = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. A URI pointing to the location of the stdout of the job's - // driver program. - string driver_output_resource_uri = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. If present, the location of miscellaneous control files - // which may be used as part of job setup and handling. If not present, - // control files may be placed in the same location as `driver_output_uri`. - string driver_control_files_uri = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. The labels to associate with this job. - // Label **keys** must contain 1 to 63 characters, and must conform to - // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt). - // Label **values** may be empty, but, if present, must contain 1 to 63 - // characters, and must conform to [RFC - // 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be - // associated with a job. - map<string, string> labels = 18 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job scheduling configuration. - JobScheduling scheduling = 20 [(google.api.field_behavior) = OPTIONAL]; - - // Output only. A UUID that uniquely identifies a job within the project - // over time. This is in contrast to a user-settable reference.job_id that - // may be reused over time. - string job_uuid = 22 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Indicates whether the job is completed. If the value is `false`, - // the job is still in progress. If `true`, the job is completed, and - // `status.state` field will indicate if it was successful, failed, - // or cancelled. - bool done = 24 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Job scheduling options. -message JobScheduling { - // Optional. Maximum number of times per hour a driver may be restarted as - // a result of driver exiting with non-zero code before job is - // reported failed. - // - // A job may be reported as thrashing if driver exits with non-zero code - // 4 times within 10 minute window. - // - // Maximum value is 10. - // - // **Note:** Currently, this restartable job option is - // not supported in Dataproc - // [workflow - // template](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template) - // jobs. - int32 max_failures_per_hour = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Maximum number of times in total a driver may be restarted as a result of - // driver exiting with non-zero code before job is reported failed. - // Maximum value is 240. - // - // **Note:** Currently, this restartable job option is - // not supported in Dataproc - // [workflow - // template](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template) - // jobs. - int32 max_failures_total = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to submit a job. -message SubmitJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job resource. - Job job = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. A unique id used to identify the request. If the server - // receives two - // [SubmitJobRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.SubmitJobRequest)s - // with the same id, then the second request will be ignored and the - // first [Job][google.cloud.dataproc.v1.Job] created and stored in the backend - // is returned. - // - // It is recommended to always set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The id must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// Job Operation metadata. -message JobMetadata { - // Output only. The job id. - string job_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Most recent job status. - JobStatus status = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Operation type. - string operation_type = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Job submission time. - google.protobuf.Timestamp start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// A request to get the resource representation for a job in a project. -message GetJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job ID. - string job_id = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to list jobs in a project. -message ListJobsRequest { - // A matcher that specifies categories of job states. - enum JobStateMatcher { - // Match all jobs, regardless of state. - ALL = 0; - - // Only match jobs in non-terminal states: PENDING, RUNNING, or - // CANCEL_PENDING. - ACTIVE = 1; - - // Only match jobs in terminal states: CANCELLED, DONE, or ERROR. - NON_ACTIVE = 2; - } - - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 6 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The number of results to return in each response. - int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The page token, returned by a previous call, to request the - // next page of results. - string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. If set, the returned jobs list includes only jobs that were - // submitted to the named cluster. - string cluster_name = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Specifies enumerated categories of jobs to list. - // (default = match ALL jobs). - // - // If `filter` is provided, `jobStateMatcher` will be ignored. - JobStateMatcher job_state_matcher = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A filter constraining the jobs to list. Filters are - // case-sensitive and have the following syntax: - // - // [field = value] AND [field [= value]] ... - // - // where **field** is `status.state` or `labels.[KEY]`, and `[KEY]` is a label - // key. **value** can be `*` to match all values. - // `status.state` can be either `ACTIVE` or `NON_ACTIVE`. - // Only the logical `AND` operator is supported; space-separated items are - // treated as having an implicit `AND` operator. - // - // Example filter: - // - // status.state = ACTIVE AND labels.env = staging AND labels.starred = * - string filter = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to update a job. -message UpdateJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job ID. - string job_id = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The changes to the job. - Job job = 4 [(google.api.field_behavior) = REQUIRED]; - - // Required. Specifies the path, relative to <code>Job</code>, of - // the field to update. For example, to update the labels of a Job the - // <code>update_mask</code> parameter would be specified as - // <code>labels</code>, and the `PATCH` request body would specify the new - // value. <strong>Note:</strong> Currently, <code>labels</code> is the only - // field that can be updated. - google.protobuf.FieldMask update_mask = 5 [(google.api.field_behavior) = REQUIRED]; -} - -// A list of jobs in a project. -message ListJobsResponse { - // Output only. Jobs list. - repeated Job jobs = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. This token is included in the response if there are more results - // to fetch. To fetch additional results, provide this value as the - // `page_token` in a subsequent <code>ListJobsRequest</code>. - string next_page_token = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to cancel a job. -message CancelJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job ID. - string job_id = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to delete a job. -message DeleteJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job ID. - string job_id = 2 [(google.api.field_behavior) = REQUIRED]; -} diff --git a/third_party/googleapis/google/cloud/dataproc/v1/operations.proto b/third_party/googleapis/google/cloud/dataproc/v1/operations.proto deleted file mode 100644 index e12bd29..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/operations.proto +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.v1; - -import "google/api/field_behavior.proto"; -import "google/protobuf/timestamp.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; -option java_multiple_files = true; -option java_outer_classname = "OperationsProto"; -option java_package = "com.google.cloud.dataproc.v1"; - -// Metadata describing the Batch operation. -message BatchOperationMetadata { - // Operation type for Batch resources - enum BatchOperationType { - // Batch operation type is unknown. - BATCH_OPERATION_TYPE_UNSPECIFIED = 0; - - // Batch operation type. - BATCH = 1; - } - - // Name of the batch for the operation. - string batch = 1; - - // Batch UUID for the operation. - string batch_uuid = 2; - - // The time when the operation was created. - google.protobuf.Timestamp create_time = 3; - - // The time when the operation finished. - google.protobuf.Timestamp done_time = 4; - - // The operation type. - BatchOperationType operation_type = 6; - - // Short description of the operation. - string description = 7; - - // Labels associated with the operation. - map<string, string> labels = 8; - - // Warnings encountered during operation execution. - repeated string warnings = 9; -} - -// The status of the operation. -message ClusterOperationStatus { - // The operation state. - enum State { - // Unused. - UNKNOWN = 0; - - // The operation has been created. - PENDING = 1; - - // The operation is running. - RUNNING = 2; - - // The operation is done; either cancelled or completed. - DONE = 3; - } - - // Output only. A message containing the operation state. - State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. A message containing the detailed operation state. - string inner_state = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. A message containing any operation metadata details. - string details = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The time this state was entered. - google.protobuf.Timestamp state_start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Metadata describing the operation. -message ClusterOperationMetadata { - // Output only. Name of the cluster for the operation. - string cluster_name = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Cluster UUID for the operation. - string cluster_uuid = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Current operation status. - ClusterOperationStatus status = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The previous operation status. - repeated ClusterOperationStatus status_history = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The operation type. - string operation_type = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Short description of operation. - string description = 12 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Labels associated with the operation - map<string, string> labels = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Errors encountered during operation execution. - repeated string warnings = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; -} diff --git a/third_party/googleapis/google/cloud/dataproc/v1/shared.proto b/third_party/googleapis/google/cloud/dataproc/v1/shared.proto deleted file mode 100644 index 1879691..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/shared.proto +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.v1; - -import "google/api/field_behavior.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; -option java_multiple_files = true; -option java_outer_classname = "SharedProto"; -option java_package = "com.google.cloud.dataproc.v1"; - -// Runtime configuration for a workload. -message RuntimeConfig { - // Optional. Version of the batch runtime. - string version = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Optional custom container image for the job runtime environment. If - // not specified, a default container image will be used. - string container_image = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, which are used to configure workload - // execution. - map<string, string> properties = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// Environment configuration for a workload. -message EnvironmentConfig { - // Optional. Execution configuration for a workload. - ExecutionConfig execution_config = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Peripherals configuration that workload has access to. - PeripheralsConfig peripherals_config = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// Execution configuration for a workload. -message ExecutionConfig { - // Optional. Service account that used to execute workload. - string service_account = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Network configuration for workload execution. - oneof network { - // Optional. Network URI to connect workload to. - string network_uri = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Subnetwork URI to connect workload to. - string subnetwork_uri = 5 [(google.api.field_behavior) = OPTIONAL]; - } - - // Optional. Tags used for network traffic control. - repeated string network_tags = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The Cloud KMS key to use for encryption. - string kms_key = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// Spark History Server configuration for the workload. -message SparkHistoryServerConfig { - // Optional. Resource name of an existing Dataproc Cluster to act as a Spark History - // Server for the workload. - // - // Example: - // - // * `projects/[project_id]/regions/[region]/clusters/[cluster_name]` - string dataproc_cluster = 1 [ - (google.api.field_behavior) = OPTIONAL - ]; -} - -// Auxiliary services configuration for a workload. -message PeripheralsConfig { - // Optional. Resource name of an existing Dataproc Metastore service. - // - // Example: - // - // * `projects/[project_id]/locations/[region]/services/[service_id]` - string metastore_service = 1 [ - (google.api.field_behavior) = OPTIONAL - ]; - - // Optional. The Spark History Server configuration for the workload. - SparkHistoryServerConfig spark_history_server_config = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// Runtime information about workload execution. -message RuntimeInfo { - // Output only. Map of remote access endpoints (such as web interfaces and APIs) to their - // URIs. - map<string, string> endpoints = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. A URI pointing to the location of the stdout and stderr of the workload. - string output_uri = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. A URI pointing to the location of the diagnostics tarball. - string diagnostic_output_uri = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// The cluster's GKE config. -message GkeClusterConfig { - // Optional. A target GKE cluster to deploy to. It must be in the same project and - // region as the Dataproc cluster (the GKE cluster can be zonal or regional). - // Format: 'projects/{project}/locations/{location}/clusters/{cluster_id}' - string gke_cluster_target = 2 [ - (google.api.field_behavior) = OPTIONAL - ]; - - // Optional. GKE NodePools where workloads will be scheduled. At least one node pool - // must be assigned the 'default' role. Each role can be given to only a - // single NodePoolTarget. All NodePools must have the same location settings. - // If a nodePoolTarget is not specified, Dataproc constructs a default - // nodePoolTarget. - repeated GkeNodePoolTarget node_pool_target = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// The configuration for running the Dataproc cluster on Kubernetes. -message KubernetesClusterConfig { - // Optional. A namespace within the Kubernetes cluster to deploy into. If this namespace - // does not exist, it is created. If it exists, Dataproc - // verifies that another Dataproc VirtualCluster is not installed - // into it. If not specified, the name of the Dataproc Cluster is used. - string kubernetes_namespace = 1 [(google.api.field_behavior) = OPTIONAL]; - - oneof config { - // Required. The configuration for running the Dataproc cluster on GKE. - GkeClusterConfig gke_cluster_config = 2 [(google.api.field_behavior) = REQUIRED]; - } - - // Optional. The software configuration for this Dataproc cluster running on Kubernetes. - KubernetesSoftwareConfig kubernetes_software_config = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// The software configuration for this Dataproc cluster running on Kubernetes. -message KubernetesSoftwareConfig { - // The components that should be installed in this Dataproc cluster. The key - // must be a string from the KubernetesComponent enumeration. The value is - // the version of the software to be installed. - // At least one entry must be specified. - map<string, string> component_version = 1; - - // The properties to set on daemon config files. - // - // Property keys are specified in `prefix:property` format, for example - // `spark:spark.kubernetes.container.image`. The following are supported - // prefixes and their mappings: - // - // * spark: `spark-defaults.conf` - // - // For more information, see [Cluster - // properties](https://cloud.google.com/dataproc/docs/concepts/cluster-properties). - map<string, string> properties = 2; -} - -// GKE NodePools that Dataproc workloads run on. -message GkeNodePoolTarget { - // `Role` specifies whose tasks will run on the NodePool. The roles can be - // specific to workloads. Exactly one GkeNodePoolTarget within the - // VirtualCluster must have 'default' role, which is used to run all workloads - // that are not associated with a NodePool. - enum Role { - // Role is unspecified. - ROLE_UNSPECIFIED = 0; - - // Any roles that are not directly assigned to a NodePool run on the - // `default` role's NodePool. - DEFAULT = 1; - - // Run controllers and webhooks. - CONTROLLER = 2; - - // Run spark driver. - SPARK_DRIVER = 3; - - // Run spark executors. - SPARK_EXECUTOR = 4; - } - - // Required. The target GKE NodePool. - // Format: - // 'projects/{project}/locations/{location}/clusters/{cluster}/nodePools/{node_pool}' - string node_pool = 1 [ - (google.api.field_behavior) = REQUIRED - ]; - - // Required. The types of role for a GKE NodePool - repeated Role roles = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The configuration for the GKE NodePool. - // - // If specified, Dataproc attempts to create a NodePool with the - // specified shape. If one with the same name already exists, it is - // verified against all specified fields. If a field differs, the - // virtual cluster creation will fail. - // - // If omitted, any NodePool with the specified name is used. If a - // NodePool with the specified name does not exist, Dataproc create a NodePool - // with default values. - GkeNodePoolConfig node_pool_config = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// The configuration of a GKE NodePool used by a [Dataproc-on-GKE -// cluster](https://cloud.google.com/dataproc/docs/concepts/jobs/dataproc-gke#create-a-dataproc-on-gke-cluster). -message GkeNodePoolConfig { - // Parameters that describe cluster nodes. - message GkeNodeConfig { - // Optional. The name of a Compute Engine [machine - // type](https://cloud.google.com/compute/docs/machine-types). - string machine_type = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Whether the nodes are created as [preemptible VM - // instances](https://cloud.google.com/compute/docs/instances/preemptible). - bool preemptible = 10 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The number of local SSD disks to attach to the node, which is limited by - // the maximum number of disks allowable per zone (see [Adding Local - // SSDs](https://cloud.google.com/compute/docs/disks/local-ssd)). - int32 local_ssd_count = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A list of [hardware - // accelerators](https://cloud.google.com/compute/docs/gpus) to attach to - // each node. - repeated GkeNodePoolAcceleratorConfig accelerators = 11 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. [Minimum CPU - // platform](https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform) - // to be used by this instance. The instance may be scheduled on the - // specified or a newer CPU platform. Specify the friendly names of CPU - // platforms, such as "Intel Haswell"` or Intel Sandy Bridge". - string min_cpu_platform = 13 [(google.api.field_behavior) = OPTIONAL]; - } - - // A GkeNodeConfigAcceleratorConfig represents a Hardware Accelerator request - // for a NodePool. - message GkeNodePoolAcceleratorConfig { - // The number of accelerator cards exposed to an instance. - int64 accelerator_count = 1; - - // The accelerator type resource namename (see GPUs on Compute Engine). - string accelerator_type = 2; - } - - // GkeNodePoolAutoscaling contains information the cluster autoscaler needs to - // adjust the size of the node pool to the current cluster usage. - message GkeNodePoolAutoscalingConfig { - // The minimum number of nodes in the NodePool. Must be >= 0 and <= - // max_node_count. - int32 min_node_count = 2; - - // The maximum number of nodes in the NodePool. Must be >= min_node_count. - // **Note:** Quota must be sufficient to scale up the cluster. - int32 max_node_count = 3; - } - - // Optional. The node pool configuration. - GkeNodeConfig config = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The list of Compute Engine - // [zones](https://cloud.google.com/compute/docs/zones#available) where - // NodePool's nodes will be located. - // - // **Note:** Currently, only one zone may be specified. - // - // If a location is not specified during NodePool creation, Dataproc will - // choose a location. - repeated string locations = 13 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The autoscaler configuration for this NodePool. The autoscaler is enabled - // only when a valid configuration is present. - GkeNodePoolAutoscalingConfig autoscaling = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// Cluster components that can be activated. -enum Component { - // Unspecified component. Specifying this will cause Cluster creation to fail. - COMPONENT_UNSPECIFIED = 0; - - // The Anaconda python distribution. The Anaconda component is not supported - // in the Dataproc - // <a - // href="/dataproc/docs/concepts/versioning/dataproc-release-2.0">2.0 - // image</a>. The 2.0 image is pre-installed with Miniconda. - ANACONDA = 5; - - // Docker - DOCKER = 13; - - // The Druid query engine. (alpha) - DRUID = 9; - - // Flink - FLINK = 14; - - // HBase. (beta) - HBASE = 11; - - // The Hive Web HCatalog (the REST service for accessing HCatalog). - HIVE_WEBHCAT = 3; - - // The Jupyter Notebook. - JUPYTER = 1; - - // The Presto query engine. - PRESTO = 6; - - // The Ranger service. - RANGER = 12; - - // The Solr service. - SOLR = 10; - - // The Zeppelin notebook. - ZEPPELIN = 4; - - // The Zookeeper service. - ZOOKEEPER = 8; -} - -// Actions in response to failure of a resource associated with a cluster. -enum FailureAction { - // When FailureAction is unspecified, failure action defaults to NO_ACTION. - FAILURE_ACTION_UNSPECIFIED = 0; - - // Take no action on failure to create a cluster resource. NO_ACTION is the - // default. - NO_ACTION = 1; - - // Delete the failed cluster resource. - DELETE = 2; -} diff --git a/third_party/googleapis/google/cloud/dataproc/v1/workflow_templates.proto b/third_party/googleapis/google/cloud/dataproc/v1/workflow_templates.proto deleted file mode 100644 index 416ba26..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/workflow_templates.proto +++ /dev/null @@ -1,807 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.v1; - -import "google/api/annotations.proto"; -import "google/api/client.proto"; -import "google/api/field_behavior.proto"; -import "google/api/resource.proto"; -import "google/cloud/dataproc/v1/clusters.proto"; -import "google/cloud/dataproc/v1/jobs.proto"; -import "google/longrunning/operations.proto"; -import "google/protobuf/duration.proto"; -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; -option java_multiple_files = true; -option java_outer_classname = "WorkflowTemplatesProto"; -option java_package = "com.google.cloud.dataproc.v1"; - -// The API interface for managing Workflow Templates in the -// Dataproc API. -service WorkflowTemplateService { - option (google.api.default_host) = "dataproc.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; - - // Creates new workflow template. - rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest) returns (WorkflowTemplate) { - option (google.api.http) = { - post: "/v1/{parent=projects/*/locations/*}/workflowTemplates" - body: "template" - additional_bindings { - post: "/v1/{parent=projects/*/regions/*}/workflowTemplates" - body: "template" - } - }; - option (google.api.method_signature) = "parent,template"; - } - - // Retrieves the latest workflow template. - // - // Can retrieve previously instantiated template by specifying optional - // version parameter. - rpc GetWorkflowTemplate(GetWorkflowTemplateRequest) returns (WorkflowTemplate) { - option (google.api.http) = { - get: "/v1/{name=projects/*/locations/*/workflowTemplates/*}" - additional_bindings { - get: "/v1/{name=projects/*/regions/*/workflowTemplates/*}" - } - }; - option (google.api.method_signature) = "name"; - } - - // Instantiates a template and begins execution. - // - // The returned Operation can be used to track execution of - // workflow by polling - // [operations.get][google.longrunning.Operations.GetOperation]. - // The Operation will complete when entire workflow is finished. - // - // The running workflow can be aborted via - // [operations.cancel][google.longrunning.Operations.CancelOperation]. - // This will cause any inflight jobs to be cancelled and workflow-owned - // clusters to be deleted. - // - // The [Operation.metadata][google.longrunning.Operation.metadata] will be - // [WorkflowMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata). - // Also see [Using - // WorkflowMetadata](https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata). - // - // On successful completion, - // [Operation.response][google.longrunning.Operation.response] will be - // [Empty][google.protobuf.Empty]. - rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/{name=projects/*/locations/*/workflowTemplates/*}:instantiate" - body: "*" - additional_bindings { - post: "/v1/{name=projects/*/regions/*/workflowTemplates/*}:instantiate" - body: "*" - } - }; - option (google.api.method_signature) = "name"; - option (google.api.method_signature) = "name,parameters"; - option (google.longrunning.operation_info) = { - response_type: "google.protobuf.Empty" - metadata_type: "WorkflowMetadata" - }; - } - - // Instantiates a template and begins execution. - // - // This method is equivalent to executing the sequence - // [CreateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.CreateWorkflowTemplate], [InstantiateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.InstantiateWorkflowTemplate], - // [DeleteWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.DeleteWorkflowTemplate]. - // - // The returned Operation can be used to track execution of - // workflow by polling - // [operations.get][google.longrunning.Operations.GetOperation]. - // The Operation will complete when entire workflow is finished. - // - // The running workflow can be aborted via - // [operations.cancel][google.longrunning.Operations.CancelOperation]. - // This will cause any inflight jobs to be cancelled and workflow-owned - // clusters to be deleted. - // - // The [Operation.metadata][google.longrunning.Operation.metadata] will be - // [WorkflowMetadata](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata). - // Also see [Using - // WorkflowMetadata](https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata). - // - // On successful completion, - // [Operation.response][google.longrunning.Operation.response] will be - // [Empty][google.protobuf.Empty]. - rpc InstantiateInlineWorkflowTemplate(InstantiateInlineWorkflowTemplateRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/{parent=projects/*/locations/*}/workflowTemplates:instantiateInline" - body: "template" - additional_bindings { - post: "/v1/{parent=projects/*/regions/*}/workflowTemplates:instantiateInline" - body: "template" - } - }; - option (google.api.method_signature) = "parent,template"; - option (google.longrunning.operation_info) = { - response_type: "google.protobuf.Empty" - metadata_type: "WorkflowMetadata" - }; - } - - // Updates (replaces) workflow template. The updated template - // must contain version that matches the current server version. - rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest) returns (WorkflowTemplate) { - option (google.api.http) = { - put: "/v1/{template.name=projects/*/locations/*/workflowTemplates/*}" - body: "template" - additional_bindings { - put: "/v1/{template.name=projects/*/regions/*/workflowTemplates/*}" - body: "template" - } - }; - option (google.api.method_signature) = "template"; - } - - // Lists workflows that match the specified filter in the request. - rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest) returns (ListWorkflowTemplatesResponse) { - option (google.api.http) = { - get: "/v1/{parent=projects/*/locations/*}/workflowTemplates" - additional_bindings { - get: "/v1/{parent=projects/*/regions/*}/workflowTemplates" - } - }; - option (google.api.method_signature) = "parent"; - } - - // Deletes a workflow template. It does not cancel in-progress workflows. - rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { - delete: "/v1/{name=projects/*/locations/*/workflowTemplates/*}" - additional_bindings { - delete: "/v1/{name=projects/*/regions/*/workflowTemplates/*}" - } - }; - option (google.api.method_signature) = "name"; - } -} - -// A Dataproc workflow template resource. -message WorkflowTemplate { - option (google.api.resource) = { - type: "dataproc.googleapis.com/WorkflowTemplate" - pattern: "projects/{project}/regions/{region}/workflowTemplates/{workflow_template}" - pattern: "projects/{project}/locations/{location}/workflowTemplates/{workflow_template}" - history: ORIGINALLY_SINGLE_PATTERN - }; - - string id = 2 [(google.api.field_behavior) = REQUIRED]; - - // Output only. The resource name of the workflow template, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.workflowTemplates`, the resource name of the - // template has the following format: - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - // - // * For `projects.locations.workflowTemplates`, the resource name of the - // template has the following format: - // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` - string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. Used to perform a consistent read-modify-write. - // - // This field should be left blank for a `CreateWorkflowTemplate` request. It - // is required for an `UpdateWorkflowTemplate` request, and must match the - // current server version. A typical update template flow would fetch the - // current template with a `GetWorkflowTemplate` request, which will return - // the current template with the `version` field filled in with the - // current server version. The user updates other fields in the template, - // then returns it as part of the `UpdateWorkflowTemplate` request. - int32 version = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Output only. The time template was created. - google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The time template was last updated. - google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. The labels to associate with this template. These labels - // will be propagated to all jobs and clusters created by the workflow - // instance. - // - // Label **keys** must contain 1 to 63 characters, and must conform to - // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt). - // - // Label **values** may be empty, but, if present, must contain 1 to 63 - // characters, and must conform to - // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt). - // - // No more than 32 labels can be associated with a template. - map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Required. WorkflowTemplate scheduling information. - WorkflowTemplatePlacement placement = 7 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Directed Acyclic Graph of Jobs to submit. - repeated OrderedJob jobs = 8 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Template parameters whose values are substituted into the - // template. Values for parameters must be provided when the template is - // instantiated. - repeated TemplateParameter parameters = 9 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Timeout duration for the DAG of jobs, expressed in seconds (see - // [JSON representation of - // duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). - // The timeout duration must be from 10 minutes ("600s") to 24 hours - // ("86400s"). The timer begins when the first job is submitted. If the - // workflow is running at the end of the timeout period, any remaining jobs - // are cancelled, the workflow is ended, and if the workflow was running on a - // [managed - // cluster](/dataproc/docs/concepts/workflows/using-workflows#configuring_or_selecting_a_cluster), - // the cluster is deleted. - google.protobuf.Duration dag_timeout = 10 [(google.api.field_behavior) = OPTIONAL]; -} - -// Specifies workflow execution target. -// -// Either `managed_cluster` or `cluster_selector` is required. -message WorkflowTemplatePlacement { - // Required. Specifies where workflow executes; either on a managed - // cluster or an existing cluster chosen by labels. - oneof placement { - // A cluster that is managed by the workflow. - ManagedCluster managed_cluster = 1; - - // Optional. A selector that chooses target cluster for jobs based - // on metadata. - // - // The selector is evaluated at the time each job is submitted. - ClusterSelector cluster_selector = 2; - } -} - -// Cluster that is managed by the workflow. -message ManagedCluster { - // Required. The cluster name prefix. A unique cluster name will be formed by - // appending a random suffix. - // - // The name must contain only lower-case letters (a-z), numbers (0-9), - // and hyphens (-). Must begin with a letter. Cannot begin or end with - // hyphen. Must consist of between 2 and 35 characters. - string cluster_name = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The cluster configuration. - ClusterConfig config = 3 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The labels to associate with this cluster. - // - // Label keys must be between 1 and 63 characters long, and must conform to - // the following PCRE regular expression: - // [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62} - // - // Label values must be between 1 and 63 characters long, and must conform to - // the following PCRE regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63} - // - // No more than 32 labels can be associated with a given cluster. - map<string, string> labels = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// A selector that chooses target cluster for jobs based on metadata. -message ClusterSelector { - // Optional. The zone where workflow process executes. This parameter does not - // affect the selection of the cluster. - // - // If unspecified, the zone of the first cluster matching the selector - // is used. - string zone = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Required. The cluster labels. Cluster must have all labels - // to match. - map<string, string> cluster_labels = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// A job executed by the workflow. -message OrderedJob { - // Required. The step id. The id must be unique among all jobs - // within the template. - // - // The step id is used as prefix for job id, as job - // `goog-dataproc-workflow-step-id` label, and in - // [prerequisiteStepIds][google.cloud.dataproc.v1.OrderedJob.prerequisite_step_ids] field from other - // steps. - // - // The id must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). Cannot begin or end with underscore - // or hyphen. Must consist of between 3 and 50 characters. - string step_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job definition. - oneof job_type { - // Optional. Job is a Hadoop job. - HadoopJob hadoop_job = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Spark job. - SparkJob spark_job = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a PySpark job. - PySparkJob pyspark_job = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Hive job. - HiveJob hive_job = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Pig job. - PigJob pig_job = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a SparkR job. - SparkRJob spark_r_job = 11 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a SparkSql job. - SparkSqlJob spark_sql_job = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Presto job. - PrestoJob presto_job = 12 [(google.api.field_behavior) = OPTIONAL]; - } - - // Optional. The labels to associate with this job. - // - // Label keys must be between 1 and 63 characters long, and must conform to - // the following regular expression: - // [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62} - // - // Label values must be between 1 and 63 characters long, and must conform to - // the following regular expression: [\p{Ll}\p{Lo}\p{N}_-]{0,63} - // - // No more than 32 labels can be associated with a given job. - map<string, string> labels = 8 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job scheduling configuration. - JobScheduling scheduling = 9 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The optional list of prerequisite job step_ids. - // If not specified, the job will start at the beginning of workflow. - repeated string prerequisite_step_ids = 10 [(google.api.field_behavior) = OPTIONAL]; -} - -// A configurable parameter that replaces one or more fields in the template. -// Parameterizable fields: -// - Labels -// - File uris -// - Job properties -// - Job arguments -// - Script variables -// - Main class (in HadoopJob and SparkJob) -// - Zone (in ClusterSelector) -message TemplateParameter { - // Required. Parameter name. - // The parameter name is used as the key, and paired with the - // parameter value, which are passed to the template when the template - // is instantiated. - // The name must contain only capital letters (A-Z), numbers (0-9), and - // underscores (_), and must not start with a number. The maximum length is - // 40 characters. - string name = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. Paths to all fields that the parameter replaces. - // A field is allowed to appear in at most one parameter's list of field - // paths. - // - // A field path is similar in syntax to a [google.protobuf.FieldMask][google.protobuf.FieldMask]. - // For example, a field path that references the zone field of a workflow - // template's cluster selector would be specified as - // `placement.clusterSelector.zone`. - // - // Also, field paths can reference fields using the following syntax: - // - // * Values in maps can be referenced by key: - // * labels['key'] - // * placement.clusterSelector.clusterLabels['key'] - // * placement.managedCluster.labels['key'] - // * placement.clusterSelector.clusterLabels['key'] - // * jobs['step-id'].labels['key'] - // - // * Jobs in the jobs list can be referenced by step-id: - // * jobs['step-id'].hadoopJob.mainJarFileUri - // * jobs['step-id'].hiveJob.queryFileUri - // * jobs['step-id'].pySparkJob.mainPythonFileUri - // * jobs['step-id'].hadoopJob.jarFileUris[0] - // * jobs['step-id'].hadoopJob.archiveUris[0] - // * jobs['step-id'].hadoopJob.fileUris[0] - // * jobs['step-id'].pySparkJob.pythonFileUris[0] - // - // * Items in repeated fields can be referenced by a zero-based index: - // * jobs['step-id'].sparkJob.args[0] - // - // * Other examples: - // * jobs['step-id'].hadoopJob.properties['key'] - // * jobs['step-id'].hadoopJob.args[0] - // * jobs['step-id'].hiveJob.scriptVariables['key'] - // * jobs['step-id'].hadoopJob.mainJarFileUri - // * placement.clusterSelector.zone - // - // It may not be possible to parameterize maps and repeated fields in their - // entirety since only individual map values and individual items in repeated - // fields can be referenced. For example, the following field paths are - // invalid: - // - // - placement.clusterSelector.clusterLabels - // - jobs['step-id'].sparkJob.args - repeated string fields = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. Brief description of the parameter. - // Must not exceed 1024 characters. - string description = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Validation rules to be applied to this parameter's value. - ParameterValidation validation = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// Configuration for parameter validation. -message ParameterValidation { - // Required. The type of validation to be performed. - oneof validation_type { - // Validation based on regular expressions. - RegexValidation regex = 1; - - // Validation based on a list of allowed values. - ValueValidation values = 2; - } -} - -// Validation based on regular expressions. -message RegexValidation { - // Required. RE2 regular expressions used to validate the parameter's value. - // The value must match the regex in its entirety (substring - // matches are not sufficient). - repeated string regexes = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// Validation based on a list of allowed values. -message ValueValidation { - // Required. List of allowed values for the parameter. - repeated string values = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// A Dataproc workflow template resource. -message WorkflowMetadata { - // The operation state. - enum State { - // Unused. - UNKNOWN = 0; - - // The operation has been created. - PENDING = 1; - - // The operation is running. - RUNNING = 2; - - // The operation is done; either cancelled or completed. - DONE = 3; - } - - // Output only. The resource name of the workflow template as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.workflowTemplates`, the resource name of the - // template has the following format: - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - // - // * For `projects.locations.workflowTemplates`, the resource name of the - // template has the following format: - // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` - string template = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The version of template at the time of - // workflow instantiation. - int32 version = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The create cluster operation metadata. - ClusterOperation create_cluster = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The workflow graph. - WorkflowGraph graph = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The delete cluster operation metadata. - ClusterOperation delete_cluster = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The workflow state. - State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The name of the target cluster. - string cluster_name = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Map from parameter names to values that were used for those parameters. - map<string, string> parameters = 8; - - // Output only. Workflow start time. - google.protobuf.Timestamp start_time = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Workflow end time. - google.protobuf.Timestamp end_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The UUID of target cluster. - string cluster_uuid = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The timeout duration for the DAG of jobs, expressed in seconds (see - // [JSON representation of - // duration](https://developers.google.com/protocol-buffers/docs/proto3#json)). - google.protobuf.Duration dag_timeout = 12 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. DAG start time, only set for workflows with [dag_timeout][google.cloud.dataproc.v1.WorkflowMetadata.dag_timeout] when DAG - // begins. - google.protobuf.Timestamp dag_start_time = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. DAG end time, only set for workflows with [dag_timeout][google.cloud.dataproc.v1.WorkflowMetadata.dag_timeout] when DAG ends. - google.protobuf.Timestamp dag_end_time = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// The cluster operation triggered by a workflow. -message ClusterOperation { - // Output only. The id of the cluster operation. - string operation_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Error, if operation failed. - string error = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Indicates the operation is done. - bool done = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// The workflow graph. -message WorkflowGraph { - // Output only. The workflow nodes. - repeated WorkflowNode nodes = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// The workflow node. -message WorkflowNode { - // The workflow node state. - enum NodeState { - // State is unspecified. - NODE_STATE_UNSPECIFIED = 0; - - // The node is awaiting prerequisite node to finish. - BLOCKED = 1; - - // The node is runnable but not running. - RUNNABLE = 2; - - // The node is running. - RUNNING = 3; - - // The node completed successfully. - COMPLETED = 4; - - // The node failed. A node can be marked FAILED because - // its ancestor or peer failed. - FAILED = 5; - } - - // Output only. The name of the node. - string step_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Node's prerequisite nodes. - repeated string prerequisite_step_ids = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The job id; populated after the node enters RUNNING state. - string job_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The node state. - NodeState state = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The error detail. - string error = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// A request to create a workflow template. -message CreateWorkflowTemplateRequest { - // Required. The resource name of the region or location, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.workflowTemplates.create`, the resource name of the - // region has the following format: - // `projects/{project_id}/regions/{region}` - // - // * For `projects.locations.workflowTemplates.create`, the resource name of - // the location has the following format: - // `projects/{project_id}/locations/{location}` - string parent = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - child_type: "dataproc.googleapis.com/WorkflowTemplate" - } - ]; - - // Required. The Dataproc workflow template to create. - WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to fetch a workflow template. -message GetWorkflowTemplateRequest { - // Required. The resource name of the workflow template, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.workflowTemplates.get`, the resource name of the - // template has the following format: - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - // - // * For `projects.locations.workflowTemplates.get`, the resource name of the - // template has the following format: - // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dataproc.googleapis.com/WorkflowTemplate" - } - ]; - - // Optional. The version of workflow template to retrieve. Only previously - // instantiated versions can be retrieved. - // - // If unspecified, retrieves the current version. - int32 version = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to instantiate a workflow template. -message InstantiateWorkflowTemplateRequest { - // Required. The resource name of the workflow template, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.workflowTemplates.instantiate`, the resource name - // of the template has the following format: - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - // - // * For `projects.locations.workflowTemplates.instantiate`, the resource name - // of the template has the following format: - // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dataproc.googleapis.com/WorkflowTemplate" - } - ]; - - // Optional. The version of workflow template to instantiate. If specified, - // the workflow will be instantiated only if the current version of - // the workflow template has the supplied version. - // - // This option cannot be used to instantiate a previous version of - // workflow template. - int32 version = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A tag that prevents multiple concurrent workflow - // instances with the same tag from running. This mitigates risk of - // concurrent instances started due to retries. - // - // It is recommended to always set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The tag must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Map from parameter names to values that should be used for those - // parameters. Values may not exceed 1000 characters. - map<string, string> parameters = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to instantiate an inline workflow template. -message InstantiateInlineWorkflowTemplateRequest { - // Required. The resource name of the region or location, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.workflowTemplates,instantiateinline`, the resource - // name of the region has the following format: - // `projects/{project_id}/regions/{region}` - // - // * For `projects.locations.workflowTemplates.instantiateinline`, the - // resource name of the location has the following format: - // `projects/{project_id}/locations/{location}` - string parent = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - child_type: "dataproc.googleapis.com/WorkflowTemplate" - } - ]; - - // Required. The workflow template to instantiate. - WorkflowTemplate template = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. A tag that prevents multiple concurrent workflow - // instances with the same tag from running. This mitigates risk of - // concurrent instances started due to retries. - // - // It is recommended to always set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The tag must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to update a workflow template. -message UpdateWorkflowTemplateRequest { - // Required. The updated workflow template. - // - // The `template.version` field must match the current version. - WorkflowTemplate template = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to list workflow templates in a project. -message ListWorkflowTemplatesRequest { - // Required. The resource name of the region or location, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.workflowTemplates,list`, the resource - // name of the region has the following format: - // `projects/{project_id}/regions/{region}` - // - // * For `projects.locations.workflowTemplates.list`, the - // resource name of the location has the following format: - // `projects/{project_id}/locations/{location}` - string parent = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - child_type: "dataproc.googleapis.com/WorkflowTemplate" - } - ]; - - // Optional. The maximum number of results to return in each response. - int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The page token, returned by a previous call, to request the - // next page of results. - string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// A response to a request to list workflow templates in a project. -message ListWorkflowTemplatesResponse { - // Output only. WorkflowTemplates list. - repeated WorkflowTemplate templates = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. This token is included in the response if there are more - // results to fetch. To fetch additional results, provide this value as the - // page_token in a subsequent <code>ListWorkflowTemplatesRequest</code>. - string next_page_token = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// A request to delete a workflow template. -// -// Currently started workflows will remain running. -message DeleteWorkflowTemplateRequest { - // Required. The resource name of the workflow template, as described - // in https://cloud.google.com/apis/design/resource_names. - // - // * For `projects.regions.workflowTemplates.delete`, the resource name - // of the template has the following format: - // `projects/{project_id}/regions/{region}/workflowTemplates/{template_id}` - // - // * For `projects.locations.workflowTemplates.instantiate`, the resource name - // of the template has the following format: - // `projects/{project_id}/locations/{location}/workflowTemplates/{template_id}` - string name = 1 [ - (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dataproc.googleapis.com/WorkflowTemplate" - } - ]; - - // Optional. The version of workflow template to delete. If specified, - // will only delete the template if the current server version matches - // specified version. - int32 version = 2 [(google.api.field_behavior) = OPTIONAL]; -} |