diff options
Diffstat (limited to 'third_party/googleapis/google/cloud/dataproc/v1/jobs.proto')
| -rw-r--r-- | third_party/googleapis/google/cloud/dataproc/v1/jobs.proto | 924 |
1 files changed, 0 insertions, 924 deletions
diff --git a/third_party/googleapis/google/cloud/dataproc/v1/jobs.proto b/third_party/googleapis/google/cloud/dataproc/v1/jobs.proto deleted file mode 100644 index 823a12c..0000000 --- a/third_party/googleapis/google/cloud/dataproc/v1/jobs.proto +++ /dev/null @@ -1,924 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package google.cloud.dataproc.v1; - -import "google/api/annotations.proto"; -import "google/api/client.proto"; -import "google/api/field_behavior.proto"; -import "google/longrunning/operations.proto"; -import "google/protobuf/empty.proto"; -import "google/protobuf/field_mask.proto"; -import "google/protobuf/timestamp.proto"; - -option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; -option java_multiple_files = true; -option java_outer_classname = "JobsProto"; -option java_package = "com.google.cloud.dataproc.v1"; - -// The JobController provides methods to manage jobs. -service JobController { - option (google.api.default_host) = "dataproc.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; - - // Submits a job to a cluster. - rpc SubmitJob(SubmitJobRequest) returns (Job) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/jobs:submit" - body: "*" - }; - option (google.api.method_signature) = "project_id,region,job"; - } - - // Submits job to a cluster. - rpc SubmitJobAsOperation(SubmitJobRequest) returns (google.longrunning.Operation) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/jobs:submitAsOperation" - body: "*" - }; - option (google.api.method_signature) = "project_id, region, job"; - option (google.longrunning.operation_info) = { - response_type: "Job" - metadata_type: "JobMetadata" - }; - } - - // Gets the resource representation for a job in a project. - rpc GetJob(GetJobRequest) returns (Job) { - option (google.api.http) = { - get: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" - }; - option (google.api.method_signature) = "project_id,region,job_id"; - } - - // Lists regions/{region}/jobs in a project. - rpc ListJobs(ListJobsRequest) returns (ListJobsResponse) { - option (google.api.http) = { - get: "/v1/projects/{project_id}/regions/{region}/jobs" - }; - option (google.api.method_signature) = "project_id,region"; - option (google.api.method_signature) = "project_id,region,filter"; - } - - // Updates a job in a project. - rpc UpdateJob(UpdateJobRequest) returns (Job) { - option (google.api.http) = { - patch: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" - body: "job" - }; - } - - // Starts a job cancellation request. To access the job resource - // after cancellation, call - // [regions/{region}/jobs.list](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs/list) - // or - // [regions/{region}/jobs.get](https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.jobs/get). - rpc CancelJob(CancelJobRequest) returns (Job) { - option (google.api.http) = { - post: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}:cancel" - body: "*" - }; - option (google.api.method_signature) = "project_id,region,job_id"; - } - - // Deletes the job from the project. If the job is active, the delete fails, - // and the response returns `FAILED_PRECONDITION`. - rpc DeleteJob(DeleteJobRequest) returns (google.protobuf.Empty) { - option (google.api.http) = { - delete: "/v1/projects/{project_id}/regions/{region}/jobs/{job_id}" - }; - option (google.api.method_signature) = "project_id,region,job_id"; - } -} - -// The runtime logging config of the job. -message LoggingConfig { - // The Log4j level for job execution. When running an - // [Apache Hive](https://hive.apache.org/) job, Cloud - // Dataproc configures the Hive client to an equivalent verbosity level. - enum Level { - // Level is unspecified. Use default level for log4j. - LEVEL_UNSPECIFIED = 0; - - // Use ALL level for log4j. - ALL = 1; - - // Use TRACE level for log4j. - TRACE = 2; - - // Use DEBUG level for log4j. - DEBUG = 3; - - // Use INFO level for log4j. - INFO = 4; - - // Use WARN level for log4j. - WARN = 5; - - // Use ERROR level for log4j. - ERROR = 6; - - // Use FATAL level for log4j. - FATAL = 7; - - // Turn off log4j. - OFF = 8; - } - - // The per-package log levels for the driver. This may include - // "root" package name to configure rootLogger. - // Examples: - // 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG' - map<string, Level> driver_log_levels = 2; -} - -// A Dataproc job for running -// [Apache Hadoop -// MapReduce](https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduceTutorial.html) -// jobs on [Apache Hadoop -// YARN](https://hadoop.apache.org/docs/r2.7.1/hadoop-yarn/hadoop-yarn-site/YARN.html). -message HadoopJob { - // Required. Indicates the location of the driver's main class. Specify - // either the jar file that contains the main class or the main class name. - // To specify both, add the jar file to `jar_file_uris`, and then specify - // the main class name in this property. - oneof driver { - // The HCFS URI of the jar file containing the main class. - // Examples: - // 'gs://foo-bucket/analytics-binaries/extract-useful-metrics-mr.jar' - // 'hdfs:/tmp/test-samples/custom-wordcount.jar' - // 'file:///home/usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar' - string main_jar_file_uri = 1; - - // The name of the driver's main class. The jar file containing the class - // must be in the default CLASSPATH or specified in `jar_file_uris`. - string main_class = 2; - } - - // Optional. The arguments to pass to the driver. Do not - // include arguments, such as `-libjars` or `-Dfoo=bar`, that can be set as - // job properties, since a collision may occur that causes an incorrect job - // submission. - repeated string args = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Jar file URIs to add to the CLASSPATHs of the - // Hadoop driver and tasks. - repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS (Hadoop Compatible Filesystem) URIs of files to be copied - // to the working directory of Hadoop drivers and distributed tasks. Useful - // for naively parallel tasks. - repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted in the working directory of - // Hadoop drivers and tasks. Supported file types: - // .jar, .tar, .tar.gz, .tgz, or .zip. - repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure Hadoop. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in /etc/hadoop/conf/*-site and - // classes in user code. - map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running [Apache Spark](http://spark.apache.org/) -// applications on YARN. -message SparkJob { - // Required. The specification of the main method to call to drive the job. - // Specify either the jar file that contains the main class or the main class - // name. To pass both a main jar and a main class in that jar, add the jar to - // `CommonJob.jar_file_uris`, and then specify the main class name in - // `main_class`. - oneof driver { - // The HCFS URI of the jar file that contains the main class. - string main_jar_file_uri = 1; - - // The name of the driver's main class. The jar file that contains the class - // must be in the default CLASSPATH or specified in `jar_file_uris`. - string main_class = 2; - } - - // Optional. The arguments to pass to the driver. Do not include arguments, - // such as `--conf`, that can be set as job properties, since a collision may - // occur that causes an incorrect job submission. - repeated string args = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the CLASSPATHs of the - // Spark driver and tasks. - repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. Useful for naively parallel tasks. - repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // .jar, .tar, .tar.gz, .tgz, and .zip. - repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure Spark. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in - // /etc/spark/conf/spark-defaults.conf and classes in user code. - map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running -// [Apache -// PySpark](https://spark.apache.org/docs/0.9.0/python-programming-guide.html) -// applications on YARN. -message PySparkJob { - // Required. The HCFS URI of the main Python file to use as the driver. Must - // be a .py file. - string main_python_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The arguments to pass to the driver. Do not include arguments, - // such as `--conf`, that can be set as job properties, since a collision may - // occur that causes an incorrect job submission. - repeated string args = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS file URIs of Python files to pass to the PySpark - // framework. Supported file types: .py, .egg, and .zip. - repeated string python_file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the CLASSPATHs of the - // Python driver and tasks. - repeated string jar_file_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. Useful for naively parallel tasks. - repeated string file_uris = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // .jar, .tar, .tar.gz, .tgz, and .zip. - repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure PySpark. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in - // /etc/spark/conf/spark-defaults.conf and classes in user code. - map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 8 [(google.api.field_behavior) = OPTIONAL]; -} - -// A list of queries to run on a cluster. -message QueryList { - // Required. The queries to execute. You do not need to end a query expression - // with a semicolon. Multiple queries can be specified in one - // string by separating each with a semicolon. Here is an example of a - // Dataproc API snippet that uses a QueryList to specify a HiveJob: - // - // "hiveJob": { - // "queryList": { - // "queries": [ - // "query1", - // "query2", - // "query3;query4", - // ] - // } - // } - repeated string queries = 1 [(google.api.field_behavior) = REQUIRED]; -} - -// A Dataproc job for running [Apache Hive](https://hive.apache.org/) -// queries on YARN. -message HiveJob { - // Required. The sequence of Hive queries to execute, specified as either - // an HCFS file URI or a list of queries. - oneof queries { - // The HCFS URI of the script that contains Hive queries. - string query_file_uri = 1; - - // A list of queries. - QueryList query_list = 2; - } - - // Optional. Whether to continue executing queries if a query fails. - // The default value is `false`. Setting to `true` can be useful when - // executing independent parallel queries. - bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Mapping of query variable names to values (equivalent to the - // Hive command: `SET name="value";`). - map<string, string> script_variables = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names and values, used to configure Hive. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml, - // /etc/hive/conf/hive-site.xml, and classes in user code. - map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the CLASSPATH of the - // Hive server and Hadoop MapReduce (MR) tasks. Can contain Hive SerDes - // and UDFs. - repeated string jar_file_uris = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running [Apache Spark -// SQL](http://spark.apache.org/sql/) queries. -message SparkSqlJob { - // Required. The sequence of Spark SQL queries to execute, specified as - // either an HCFS file URI or as a list of queries. - oneof queries { - // The HCFS URI of the script that contains SQL queries. - string query_file_uri = 1; - - // A list of queries. - QueryList query_list = 2; - } - - // Optional. Mapping of query variable names to values (equivalent to the - // Spark SQL command: SET `name="value";`). - map<string, string> script_variables = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure - // Spark SQL's SparkConf. Properties that conflict with values set by the - // Dataproc API may be overwritten. - map<string, string> properties = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH. - repeated string jar_file_uris = 56 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running [Apache Pig](https://pig.apache.org/) -// queries on YARN. -message PigJob { - // Required. The sequence of Pig queries to execute, specified as an HCFS - // file URI or a list of queries. - oneof queries { - // The HCFS URI of the script that contains the Pig queries. - string query_file_uri = 1; - - // A list of queries. - QueryList query_list = 2; - } - - // Optional. Whether to continue executing queries if a query fails. - // The default value is `false`. Setting to `true` can be useful when - // executing independent parallel queries. - bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Mapping of query variable names to values (equivalent to the Pig - // command: `name=[value]`). - map<string, string> script_variables = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure Pig. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in /etc/hadoop/conf/*-site.xml, - // /etc/pig/conf/pig.properties, and classes in user code. - map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of jar files to add to the CLASSPATH of - // the Pig Client and Hadoop MapReduce (MR) tasks. Can contain Pig UDFs. - repeated string jar_file_uris = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running -// [Apache SparkR](https://spark.apache.org/docs/latest/sparkr.html) -// applications on YARN. -message SparkRJob { - // Required. The HCFS URI of the main R file to use as the driver. - // Must be a .R file. - string main_r_file_uri = 1 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The arguments to pass to the driver. Do not include arguments, - // such as `--conf`, that can be set as job properties, since a collision may - // occur that causes an incorrect job submission. - repeated string args = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of files to be placed in the working directory of - // each executor. Useful for naively parallel tasks. - repeated string file_uris = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. HCFS URIs of archives to be extracted into the working directory - // of each executor. Supported file types: - // .jar, .tar, .tar.gz, .tgz, and .zip. - repeated string archive_uris = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values, used to configure SparkR. - // Properties that conflict with values set by the Dataproc API may be - // overwritten. Can include properties set in - // /etc/spark/conf/spark-defaults.conf and classes in user code. - map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 6 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job for running [Presto](https://prestosql.io/) queries. -// **IMPORTANT**: The [Dataproc Presto Optional -// Component](https://cloud.google.com/dataproc/docs/concepts/components/presto) -// must be enabled when the cluster is created to submit a Presto job to the -// cluster. -message PrestoJob { - // Required. The sequence of Presto queries to execute, specified as - // either an HCFS file URI or as a list of queries. - oneof queries { - // The HCFS URI of the script that contains SQL queries. - string query_file_uri = 1; - - // A list of queries. - QueryList query_list = 2; - } - - // Optional. Whether to continue executing queries if a query fails. - // The default value is `false`. Setting to `true` can be useful when - // executing independent parallel queries. - bool continue_on_failure = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The format in which query output will be displayed. See the - // Presto documentation for supported output formats - string output_format = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Presto client tags to attach to this query - repeated string client_tags = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A mapping of property names to values. Used to set Presto - // [session properties](https://prestodb.io/docs/current/sql/set-session.html) - // Equivalent to using the --session flag in the Presto CLI - map<string, string> properties = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The runtime log config for job execution. - LoggingConfig logging_config = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// Dataproc job config. -message JobPlacement { - // Required. The name of the cluster where the job will be submitted. - string cluster_name = 1 [(google.api.field_behavior) = REQUIRED]; - - // Output only. A cluster UUID generated by the Dataproc service when - // the job is submitted. - string cluster_uuid = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. Cluster labels to identify a cluster where the job will be submitted. - map<string, string> cluster_labels = 3 [(google.api.field_behavior) = OPTIONAL]; -} - -// Dataproc job status. -message JobStatus { - // The job state. - enum State { - // The job state is unknown. - STATE_UNSPECIFIED = 0; - - // The job is pending; it has been submitted, but is not yet running. - PENDING = 1; - - // Job has been received by the service and completed initial setup; - // it will soon be submitted to the cluster. - SETUP_DONE = 8; - - // The job is running on the cluster. - RUNNING = 2; - - // A CancelJob request has been received, but is pending. - CANCEL_PENDING = 3; - - // Transient in-flight resources have been canceled, and the request to - // cancel the running job has been issued to the cluster. - CANCEL_STARTED = 7; - - // The job cancellation was successful. - CANCELLED = 4; - - // The job has completed successfully. - DONE = 5; - - // The job has completed, but encountered an error. - ERROR = 6; - - // Job attempt has failed. The detail field contains failure details for - // this attempt. - // - // Applies to restartable jobs only. - ATTEMPT_FAILURE = 9; - } - - // The job substate. - enum Substate { - // The job substate is unknown. - UNSPECIFIED = 0; - - // The Job is submitted to the agent. - // - // Applies to RUNNING state. - SUBMITTED = 1; - - // The Job has been received and is awaiting execution (it may be waiting - // for a condition to be met). See the "details" field for the reason for - // the delay. - // - // Applies to RUNNING state. - QUEUED = 2; - - // The agent-reported status is out of date, which may be caused by a - // loss of communication between the agent and Dataproc. If the - // agent does not send a timely update, the job will fail. - // - // Applies to RUNNING state. - STALE_STATUS = 3; - } - - // Output only. A state message specifying the overall job state. - State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. Output only. Job state details, such as an error - // description if the state is <code>ERROR</code>. - string details = 2 [ - (google.api.field_behavior) = OUTPUT_ONLY, - (google.api.field_behavior) = OPTIONAL - ]; - - // Output only. The time when this state was entered. - google.protobuf.Timestamp state_start_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Additional state information, which includes - // status reported by the agent. - Substate substate = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Encapsulates the full scoping used to reference a job. -message JobReference { - // Optional. The ID of the Google Cloud Platform project that the job belongs to. If - // specified, must match the request project ID. - string project_id = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The job ID, which must be unique within the project. - // - // The ID must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), or hyphens (-). The maximum length is 100 characters. - // - // If not specified by the caller, the job ID will be provided by the server. - string job_id = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// A YARN application created by a job. Application information is a subset of -// <code>org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto</code>. -// -// **Beta Feature**: This report is available for testing purposes only. It may -// be changed before final release. -message YarnApplication { - // The application state, corresponding to - // <code>YarnProtos.YarnApplicationStateProto</code>. - enum State { - // Status is unspecified. - STATE_UNSPECIFIED = 0; - - // Status is NEW. - NEW = 1; - - // Status is NEW_SAVING. - NEW_SAVING = 2; - - // Status is SUBMITTED. - SUBMITTED = 3; - - // Status is ACCEPTED. - ACCEPTED = 4; - - // Status is RUNNING. - RUNNING = 5; - - // Status is FINISHED. - FINISHED = 6; - - // Status is FAILED. - FAILED = 7; - - // Status is KILLED. - KILLED = 8; - } - - // Required. The application name. - string name = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The application state. - State state = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The numerical progress of the application, from 1 to 100. - float progress = 3 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The HTTP URL of the ApplicationMaster, HistoryServer, or - // TimelineServer that provides application-specific information. The URL uses - // the internal hostname, and requires a proxy server for resolution and, - // possibly, access. - string tracking_url = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// A Dataproc job resource. -message Job { - // Optional. The fully qualified reference to the job, which can be used to - // obtain the equivalent REST path of the job resource. If this property - // is not specified when a job is created, the server generates a - // <code>job_id</code>. - JobReference reference = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Required. Job information, including how, when, and where to - // run the job. - JobPlacement placement = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The application/framework-specific portion of the job. - oneof type_job { - // Optional. Job is a Hadoop job. - HadoopJob hadoop_job = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Spark job. - SparkJob spark_job = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a PySpark job. - PySparkJob pyspark_job = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Hive job. - HiveJob hive_job = 6 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Pig job. - PigJob pig_job = 7 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a SparkR job. - SparkRJob spark_r_job = 21 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a SparkSql job. - SparkSqlJob spark_sql_job = 12 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job is a Presto job. - PrestoJob presto_job = 23 [(google.api.field_behavior) = OPTIONAL]; - } - - // Output only. The job status. Additional application-specific - // status information may be contained in the <code>type_job</code> - // and <code>yarn_applications</code> fields. - JobStatus status = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The previous job status. - repeated JobStatus status_history = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. The collection of YARN applications spun up by this job. - // - // **Beta** Feature: This report is available for testing purposes only. It - // may be changed before final release. - repeated YarnApplication yarn_applications = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. A URI pointing to the location of the stdout of the job's - // driver program. - string driver_output_resource_uri = 17 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. If present, the location of miscellaneous control files - // which may be used as part of job setup and handling. If not present, - // control files may be placed in the same location as `driver_output_uri`. - string driver_control_files_uri = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. The labels to associate with this job. - // Label **keys** must contain 1 to 63 characters, and must conform to - // [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt). - // Label **values** may be empty, but, if present, must contain 1 to 63 - // characters, and must conform to [RFC - // 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be - // associated with a job. - map<string, string> labels = 18 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Job scheduling configuration. - JobScheduling scheduling = 20 [(google.api.field_behavior) = OPTIONAL]; - - // Output only. A UUID that uniquely identifies a job within the project - // over time. This is in contrast to a user-settable reference.job_id that - // may be reused over time. - string job_uuid = 22 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Indicates whether the job is completed. If the value is `false`, - // the job is still in progress. If `true`, the job is completed, and - // `status.state` field will indicate if it was successful, failed, - // or cancelled. - bool done = 24 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// Job scheduling options. -message JobScheduling { - // Optional. Maximum number of times per hour a driver may be restarted as - // a result of driver exiting with non-zero code before job is - // reported failed. - // - // A job may be reported as thrashing if driver exits with non-zero code - // 4 times within 10 minute window. - // - // Maximum value is 10. - // - // **Note:** Currently, this restartable job option is - // not supported in Dataproc - // [workflow - // template](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template) - // jobs. - int32 max_failures_per_hour = 1 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Maximum number of times in total a driver may be restarted as a result of - // driver exiting with non-zero code before job is reported failed. - // Maximum value is 240. - // - // **Note:** Currently, this restartable job option is - // not supported in Dataproc - // [workflow - // template](https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template) - // jobs. - int32 max_failures_total = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to submit a job. -message SubmitJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job resource. - Job job = 2 [(google.api.field_behavior) = REQUIRED]; - - // Optional. A unique id used to identify the request. If the server - // receives two - // [SubmitJobRequest](https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#google.cloud.dataproc.v1.SubmitJobRequest)s - // with the same id, then the second request will be ignored and the - // first [Job][google.cloud.dataproc.v1.Job] created and stored in the backend - // is returned. - // - // It is recommended to always set this value to a - // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). - // - // The id must contain only letters (a-z, A-Z), numbers (0-9), - // underscores (_), and hyphens (-). The maximum length is 40 characters. - string request_id = 4 [(google.api.field_behavior) = OPTIONAL]; -} - -// Job Operation metadata. -message JobMetadata { - // Output only. The job id. - string job_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Most recent job status. - JobStatus status = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Operation type. - string operation_type = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Output only. Job submission time. - google.protobuf.Timestamp start_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; -} - -// A request to get the resource representation for a job in a project. -message GetJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job ID. - string job_id = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to list jobs in a project. -message ListJobsRequest { - // A matcher that specifies categories of job states. - enum JobStateMatcher { - // Match all jobs, regardless of state. - ALL = 0; - - // Only match jobs in non-terminal states: PENDING, RUNNING, or - // CANCEL_PENDING. - ACTIVE = 1; - - // Only match jobs in terminal states: CANCELLED, DONE, or ERROR. - NON_ACTIVE = 2; - } - - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 6 [(google.api.field_behavior) = REQUIRED]; - - // Optional. The number of results to return in each response. - int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. The page token, returned by a previous call, to request the - // next page of results. - string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. If set, the returned jobs list includes only jobs that were - // submitted to the named cluster. - string cluster_name = 4 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. Specifies enumerated categories of jobs to list. - // (default = match ALL jobs). - // - // If `filter` is provided, `jobStateMatcher` will be ignored. - JobStateMatcher job_state_matcher = 5 [(google.api.field_behavior) = OPTIONAL]; - - // Optional. A filter constraining the jobs to list. Filters are - // case-sensitive and have the following syntax: - // - // [field = value] AND [field [= value]] ... - // - // where **field** is `status.state` or `labels.[KEY]`, and `[KEY]` is a label - // key. **value** can be `*` to match all values. - // `status.state` can be either `ACTIVE` or `NON_ACTIVE`. - // Only the logical `AND` operator is supported; space-separated items are - // treated as having an implicit `AND` operator. - // - // Example filter: - // - // status.state = ACTIVE AND labels.env = staging AND labels.starred = * - string filter = 7 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to update a job. -message UpdateJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 2 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job ID. - string job_id = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The changes to the job. - Job job = 4 [(google.api.field_behavior) = REQUIRED]; - - // Required. Specifies the path, relative to <code>Job</code>, of - // the field to update. For example, to update the labels of a Job the - // <code>update_mask</code> parameter would be specified as - // <code>labels</code>, and the `PATCH` request body would specify the new - // value. <strong>Note:</strong> Currently, <code>labels</code> is the only - // field that can be updated. - google.protobuf.FieldMask update_mask = 5 [(google.api.field_behavior) = REQUIRED]; -} - -// A list of jobs in a project. -message ListJobsResponse { - // Output only. Jobs list. - repeated Job jobs = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - - // Optional. This token is included in the response if there are more results - // to fetch. To fetch additional results, provide this value as the - // `page_token` in a subsequent <code>ListJobsRequest</code>. - string next_page_token = 2 [(google.api.field_behavior) = OPTIONAL]; -} - -// A request to cancel a job. -message CancelJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job ID. - string job_id = 2 [(google.api.field_behavior) = REQUIRED]; -} - -// A request to delete a job. -message DeleteJobRequest { - // Required. The ID of the Google Cloud Platform project that the job - // belongs to. - string project_id = 1 [(google.api.field_behavior) = REQUIRED]; - - // Required. The Dataproc region in which to handle the request. - string region = 3 [(google.api.field_behavior) = REQUIRED]; - - // Required. The job ID. - string job_id = 2 [(google.api.field_behavior) = REQUIRED]; -} |