diff options
Diffstat (limited to 'third_party/googleapis/google/dataflow')
11 files changed, 3680 insertions, 0 deletions
diff --git a/third_party/googleapis/google/dataflow/BUILD.bazel b/third_party/googleapis/google/dataflow/BUILD.bazel new file mode 100644 index 0000000..dbdd328 --- /dev/null +++ b/third_party/googleapis/google/dataflow/BUILD.bazel @@ -0,0 +1,41 @@ +# This build file includes a target for the Ruby wrapper library for +# google-cloud-dataflow. + +# This is an API workspace, having public visibility by default makes perfect sense. +package(default_visibility = ["//visibility:public"]) + +# Export yaml configs. +exports_files(glob(["*.yaml"])) + +load( + "@com_google_googleapis_imports//:imports.bzl", + "ruby_cloud_gapic_library", + "ruby_gapic_assembly_pkg", +) + +# Generates a Ruby wrapper client for dataflow. +# Ruby wrapper clients are versionless, but are generated from source protos +# for a particular service version, v1beta3 in this case. +ruby_cloud_gapic_library( + name = "dataflow_ruby_wrapper", + srcs = ["//google/dataflow/v1beta3:dataflow_proto_with_info"], + extra_protoc_parameters = [ + "ruby-cloud-gem-name=google-cloud-dataflow", + "ruby-cloud-env-prefix=DATAFLOW", + "ruby-cloud-wrapper-of=v1beta3:0.0", + "ruby-cloud-product-url=https://cloud.google.com/dataflow", + "ruby-cloud-api-id=dataflow.googleapis.com", + "ruby-cloud-api-shortname=dataflow", + "ruby-cloud-service-override=JobsV1Beta3=Jobs;MessagesV1Beta3=Messages;MetricsV1Beta3=Metrics;SnapshotsV1Beta3=Snapshots", + ], + ruby_cloud_description = "Dataflow is a managed service for executing a wide variety of data processing patterns.", + ruby_cloud_title = "Dataflow", +) + +# Open Source package. +ruby_gapic_assembly_pkg( + name = "google-cloud-dataflow-ruby", + deps = [ + ":dataflow_ruby_wrapper", + ], +) diff --git a/third_party/googleapis/google/dataflow/v1beta3/BUILD.bazel b/third_party/googleapis/google/dataflow/v1beta3/BUILD.bazel new file mode 100644 index 0000000..1600127 --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/BUILD.bazel @@ -0,0 +1,402 @@ +# This file was automatically generated by BuildFileGenerator +# https://github.com/googleapis/rules_gapic/tree/master/bazel + +# Most of the manual changes to this file will be overwritten. +# It's **only** allowed to change the following rule attribute values: +# - names of *_gapic_assembly_* rules +# - certain parameters of *_gapic_library rules, including but not limited to: +# * extra_protoc_parameters +# * extra_protoc_file_parameters +# The complete list of preserved parameters can be found in the source code. + +# This is an API workspace, having public visibility by default makes perfect sense. +package(default_visibility = ["//visibility:public"]) + +############################################################################## +# Common +############################################################################## +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@com_google_googleapis_imports//:imports.bzl", "proto_library_with_info") + +proto_library( + name = "dataflow_proto", + srcs = [ + "environment.proto", + "jobs.proto", + "messages.proto", + "metrics.proto", + "snapshots.proto", + "streaming.proto", + "templates.proto", + ], + deps = [ + "//google/api:annotations_proto", + "//google/api:client_proto", + "//google/api:field_behavior_proto", + "//google/rpc:status_proto", + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +proto_library_with_info( + name = "dataflow_proto_with_info", + deps = [ + ":dataflow_proto", + "//google/cloud:common_resources_proto", + ], +) + +############################################################################## +# Java +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "java_gapic_assembly_gradle_pkg", + "java_gapic_library", + "java_gapic_test", + "java_grpc_library", + "java_proto_library", +) + +java_proto_library( + name = "dataflow_java_proto", + deps = [":dataflow_proto"], +) + +java_grpc_library( + name = "dataflow_java_grpc", + srcs = [":dataflow_proto"], + deps = [":dataflow_java_proto"], +) + +java_gapic_library( + name = "dataflow_java_gapic", + srcs = [":dataflow_proto_with_info"], + gapic_yaml = None, + grpc_service_config = "dataflow_grpc_service_config.json", + service_yaml = "dataflow_v1beta3.yaml", + test_deps = [ + ":dataflow_java_grpc", + ], + transport = "grpc+rest", + deps = [ + ":dataflow_java_proto", + "//google/api:api_java_proto", + ], +) + +java_gapic_test( + name = "dataflow_java_gapic_test_suite", + test_classes = [ + "com.google.dataflow.v1beta3.FlexTemplatesServiceClientHttpJsonTest", + "com.google.dataflow.v1beta3.FlexTemplatesServiceClientTest", + "com.google.dataflow.v1beta3.JobsV1Beta3ClientHttpJsonTest", + "com.google.dataflow.v1beta3.JobsV1Beta3ClientTest", + "com.google.dataflow.v1beta3.MessagesV1Beta3ClientHttpJsonTest", + "com.google.dataflow.v1beta3.MessagesV1Beta3ClientTest", + "com.google.dataflow.v1beta3.MetricsV1Beta3ClientHttpJsonTest", + "com.google.dataflow.v1beta3.MetricsV1Beta3ClientTest", + "com.google.dataflow.v1beta3.SnapshotsV1Beta3ClientHttpJsonTest", + "com.google.dataflow.v1beta3.SnapshotsV1Beta3ClientTest", + "com.google.dataflow.v1beta3.TemplatesServiceClientHttpJsonTest", + "com.google.dataflow.v1beta3.TemplatesServiceClientTest", + ], + runtime_deps = [":dataflow_java_gapic_test"], +) + +# Open Source Packages +java_gapic_assembly_gradle_pkg( + name = "google-cloud-dataflow-v1beta3-java", + transport = "grpc+rest", + deps = [ + ":dataflow_java_gapic", + ":dataflow_java_grpc", + ":dataflow_java_proto", + ":dataflow_proto", + ], +) + +############################################################################## +# Go +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "go_gapic_assembly_pkg", + "go_gapic_library", + "go_proto_library", + "go_test", +) + +go_proto_library( + name = "dataflow_go_proto", + compilers = ["@io_bazel_rules_go//proto:go_grpc"], + importpath = "google.golang.org/genproto/googleapis/dataflow/v1beta3", + protos = [":dataflow_proto"], + deps = [ + "//google/api:annotations_go_proto", + "//google/rpc:status_go_proto", + ], +) + +go_gapic_library( + name = "dataflow_go_gapic", + srcs = [":dataflow_proto_with_info"], + grpc_service_config = "dataflow_grpc_service_config.json", + importpath = "cloud.google.com/go/dataflow/apiv1beta3;dataflow", + metadata = True, + service_yaml = "dataflow_v1beta3.yaml", + transport = "grpc+rest", + deps = [ + ":dataflow_go_proto", + "@io_bazel_rules_go//proto/wkt:duration_go_proto", + ], +) + +go_test( + name = "dataflow_go_gapic_test", + srcs = [":dataflow_go_gapic_srcjar_test"], + embed = [":dataflow_go_gapic"], + importpath = "cloud.google.com/go/dataflow/apiv1beta3", +) + +# Open Source Packages +go_gapic_assembly_pkg( + name = "gapi-cloud-dataflow-v1beta3-go", + deps = [ + ":dataflow_go_gapic", + ":dataflow_go_gapic_srcjar-metadata.srcjar", + ":dataflow_go_gapic_srcjar-test.srcjar", + ":dataflow_go_proto", + ], +) + +############################################################################## +# Python +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "py_gapic_assembly_pkg", + "py_gapic_library", + "py_test", +) + +py_gapic_library( + name = "dataflow_py_gapic", + srcs = [":dataflow_proto"], + grpc_service_config = "dataflow_grpc_service_config.json", + opt_args = [ + "python-gapic-name=dataflow", + "python-gapic-namespace=google.cloud", + "warehouse-package-name=google-cloud-dataflow-client", + ], + transport = "grpc+rest", +) + +py_test( + name = "dataflow_py_gapic_test", + srcs = [ + "dataflow_py_gapic_pytest.py", + "dataflow_py_gapic_test.py", + ], + legacy_create_init = False, + deps = [":dataflow_py_gapic"], +) + +# Open Source Packages +py_gapic_assembly_pkg( + name = "dataflow-v1beta3-py", + deps = [ + ":dataflow_py_gapic", + ], +) + +############################################################################## +# PHP +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "php_gapic_assembly_pkg", + "php_gapic_library", + "php_grpc_library", + "php_proto_library", +) + +php_proto_library( + name = "dataflow_php_proto", + deps = [":dataflow_proto"], +) + +php_grpc_library( + name = "dataflow_php_grpc", + srcs = [":dataflow_proto"], + deps = [":dataflow_php_proto"], +) + +php_gapic_library( + name = "dataflow_php_gapic", + srcs = [":dataflow_proto_with_info"], + grpc_service_config = "dataflow_grpc_service_config.json", + service_yaml = "dataflow_v1beta3.yaml", + deps = [ + ":dataflow_php_grpc", + ":dataflow_php_proto", + ], +) + +# Open Source Packages +php_gapic_assembly_pkg( + name = "google-cloud-dataflow-v1beta3-php", + deps = [ + ":dataflow_php_gapic", + ":dataflow_php_grpc", + ":dataflow_php_proto", + ], +) + +############################################################################## +# Node.js +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "nodejs_gapic_assembly_pkg", + "nodejs_gapic_library", +) + +nodejs_gapic_library( + name = "dataflow_nodejs_gapic", + package_name = "@google-cloud/dataflow", + src = ":dataflow_proto_with_info", + extra_protoc_parameters = ["metadata"], + grpc_service_config = "dataflow_grpc_service_config.json", + package = "google.dataflow.v1beta3", + service_yaml = "dataflow_v1beta3.yaml", + deps = [], +) + +nodejs_gapic_assembly_pkg( + name = "dataflow-v1beta3-nodejs", + deps = [ + ":dataflow_nodejs_gapic", + ":dataflow_proto", + ], +) + +############################################################################## +# Ruby +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "ruby_cloud_gapic_library", + "ruby_gapic_assembly_pkg", + "ruby_grpc_library", + "ruby_proto_library", +) + +ruby_proto_library( + name = "dataflow_ruby_proto", + deps = [":dataflow_proto"], +) + +ruby_grpc_library( + name = "dataflow_ruby_grpc", + srcs = [":dataflow_proto"], + deps = [":dataflow_ruby_proto"], +) + +ruby_cloud_gapic_library( + name = "dataflow_ruby_gapic", + srcs = [":dataflow_proto_with_info"], + extra_protoc_parameters = [ + "ruby-cloud-api-id=dataflow.googleapis.com", + "ruby-cloud-api-shortname=dataflow", + "ruby-cloud-env-prefix=DATAFLOW", + "ruby-cloud-gem-name=google-cloud-dataflow-v1beta3", + "ruby-cloud-product-url=https://cloud.google.com/dataflow", + "ruby-cloud-service-override=JobsV1Beta3=Jobs;MessagesV1Beta3=Messages;MetricsV1Beta3=Metrics;SnapshotsV1Beta3=Snapshots", + ], + grpc_service_config = "dataflow_grpc_service_config.json", + ruby_cloud_description = "Dataflow is a managed service for executing a wide variety of data processing patterns.", + ruby_cloud_title = "Dataflow V1beta3", + deps = [ + ":dataflow_ruby_grpc", + ":dataflow_ruby_proto", + ], +) + +# Open Source Packages +ruby_gapic_assembly_pkg( + name = "google-cloud-dataflow-v1beta3-ruby", + deps = [ + ":dataflow_ruby_gapic", + ":dataflow_ruby_grpc", + ":dataflow_ruby_proto", + ], +) + +############################################################################## +# C# +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "csharp_gapic_assembly_pkg", + "csharp_gapic_library", + "csharp_grpc_library", + "csharp_proto_library", +) + +csharp_proto_library( + name = "dataflow_csharp_proto", + deps = [":dataflow_proto"], +) + +csharp_grpc_library( + name = "dataflow_csharp_grpc", + srcs = [":dataflow_proto"], + deps = [":dataflow_csharp_proto"], +) + +csharp_gapic_library( + name = "dataflow_csharp_gapic", + srcs = [":dataflow_proto_with_info"], + common_resources_config = "@gax_dotnet//:Google.Api.Gax/ResourceNames/CommonResourcesConfig.json", + grpc_service_config = "dataflow_grpc_service_config.json", + service_yaml = "dataflow_v1beta3.yaml", + deps = [ + ":dataflow_csharp_grpc", + ":dataflow_csharp_proto", + ], +) + +# Open Source Packages +csharp_gapic_assembly_pkg( + name = "google-cloud-dataflow-v1beta3-csharp", + deps = [ + ":dataflow_csharp_gapic", + ":dataflow_csharp_grpc", + ":dataflow_csharp_proto", + ], +) + +############################################################################## +# C++ +############################################################################## +load( + "@com_google_googleapis_imports//:imports.bzl", + "cc_grpc_library", + "cc_proto_library", +) + +cc_proto_library( + name = "dataflow_cc_proto", + deps = [":dataflow_proto"], +) + +cc_grpc_library( + name = "dataflow_cc_grpc", + srcs = [":dataflow_proto"], + grpc_only = True, + deps = [":dataflow_cc_proto"], +) diff --git a/third_party/googleapis/google/dataflow/v1beta3/dataflow_grpc_service_config.json b/third_party/googleapis/google/dataflow/v1beta3/dataflow_grpc_service_config.json new file mode 100644 index 0000000..0a6e2cd --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/dataflow_grpc_service_config.json @@ -0,0 +1,24 @@ +{ + "methodConfig": [ + { + "name": [ + { + "service": "google.dataflow.v1beta3.JobsV1Beta3" + }, + { + "service": "google.dataflow.v1beta3.MessagesV1Beta3" + }, + { + "service": "google.dataflow.v1beta3.MetricsV1Beta3" + }, + { + "service": "google.dataflow.v1beta3.SnapshotsV1Beta3" + }, + { + "service": "google.dataflow.v1beta3.TemplatesService" + } + ], + "timeout": "60s" + } + ] +} diff --git a/third_party/googleapis/google/dataflow/v1beta3/dataflow_v1beta3.yaml b/third_party/googleapis/google/dataflow/v1beta3/dataflow_v1beta3.yaml new file mode 100644 index 0000000..6699f41 --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/dataflow_v1beta3.yaml @@ -0,0 +1,75 @@ +type: google.api.Service +config_version: 3 +name: dataflow.googleapis.com +title: Dataflow API + +apis: +- name: google.dataflow.v1beta3.FlexTemplatesService +- name: google.dataflow.v1beta3.JobsV1Beta3 +- name: google.dataflow.v1beta3.MessagesV1Beta3 +- name: google.dataflow.v1beta3.MetricsV1Beta3 +- name: google.dataflow.v1beta3.SnapshotsV1Beta3 +- name: google.dataflow.v1beta3.TemplatesService + +documentation: + summary: Manages Google Cloud Dataflow projects on Google Cloud Platform. + +backend: + rules: + - selector: google.dataflow.v1beta3.FlexTemplatesService.LaunchFlexTemplate + deadline: 30.0 + - selector: 'google.dataflow.v1beta3.JobsV1Beta3.*' + deadline: 30.0 + - selector: google.dataflow.v1beta3.MessagesV1Beta3.ListJobMessages + deadline: 30.0 + - selector: 'google.dataflow.v1beta3.MetricsV1Beta3.*' + deadline: 30.0 + - selector: 'google.dataflow.v1beta3.SnapshotsV1Beta3.*' + deadline: 30.0 + - selector: 'google.dataflow.v1beta3.TemplatesService.*' + deadline: 30.0 + +authentication: + rules: + - selector: google.dataflow.v1beta3.FlexTemplatesService.LaunchFlexTemplate + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/compute, + https://www.googleapis.com/auth/compute.readonly, + https://www.googleapis.com/auth/userinfo.email + - selector: 'google.dataflow.v1beta3.JobsV1Beta3.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/compute, + https://www.googleapis.com/auth/compute.readonly, + https://www.googleapis.com/auth/userinfo.email + - selector: google.dataflow.v1beta3.MessagesV1Beta3.ListJobMessages + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/compute, + https://www.googleapis.com/auth/compute.readonly, + https://www.googleapis.com/auth/userinfo.email + - selector: 'google.dataflow.v1beta3.MetricsV1Beta3.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/compute, + https://www.googleapis.com/auth/compute.readonly, + https://www.googleapis.com/auth/userinfo.email + - selector: 'google.dataflow.v1beta3.SnapshotsV1Beta3.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/compute, + https://www.googleapis.com/auth/compute.readonly, + https://www.googleapis.com/auth/userinfo.email + - selector: 'google.dataflow.v1beta3.TemplatesService.*' + oauth: + canonical_scopes: |- + https://www.googleapis.com/auth/cloud-platform, + https://www.googleapis.com/auth/compute, + https://www.googleapis.com/auth/compute.readonly, + https://www.googleapis.com/auth/userinfo.email diff --git a/third_party/googleapis/google/dataflow/v1beta3/environment.proto b/third_party/googleapis/google/dataflow/v1beta3/environment.proto new file mode 100644 index 0000000..04ad7d7 --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/environment.proto @@ -0,0 +1,554 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.dataflow.v1beta3; + +import "google/api/field_behavior.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3"; +option go_package = "google.golang.org/genproto/googleapis/dataflow/v1beta3;dataflow"; +option java_multiple_files = true; +option java_outer_classname = "EnvironmentProto"; +option java_package = "com.google.dataflow.v1beta3"; +option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3"; +option ruby_package = "Google::Cloud::Dataflow::V1beta3"; + +// Describes the environment in which a Dataflow Job runs. +message Environment { + // The prefix of the resources the system should use for temporary + // storage. The system will append the suffix "/temp-{JOBNAME} to + // this resource prefix, where {JOBNAME} is the value of the + // job_name field. The resulting bucket and object prefix is used + // as the prefix of the resources used to store temporary data + // needed during the job execution. NOTE: This will override the + // value in taskrunner_settings. + // The supported resource type is: + // + // Google Cloud Storage: + // + // storage.googleapis.com/{bucket}/{object} + // bucket.storage.googleapis.com/{object} + string temp_storage_prefix = 1; + + // The type of cluster manager API to use. If unknown or + // unspecified, the service will attempt to choose a reasonable + // default. This should be in the form of the API service name, + // e.g. "compute.googleapis.com". + string cluster_manager_api_service = 2; + + // The list of experiments to enable. This field should be used for SDK + // related experiments and not for service related experiments. The proper + // field for service related experiments is service_options. + repeated string experiments = 3; + + // The list of service options to enable. This field should be used for + // service related experiments only. These experiments, when graduating to GA, + // should be replaced by dedicated fields or become default (i.e. always on). + repeated string service_options = 16; + + // If set, contains the Cloud KMS key identifier used to encrypt data + // at rest, AKA a Customer Managed Encryption Key (CMEK). + // + // Format: + // projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY + string service_kms_key_name = 12; + + // The worker pools. At least one "harness" worker pool must be + // specified in order for the job to have workers. + repeated WorkerPool worker_pools = 4; + + // A description of the process that generated the request. + google.protobuf.Struct user_agent = 5; + + // A structure describing which components and their versions of the service + // are required in order to run the job. + google.protobuf.Struct version = 6; + + // The dataset for the current project where various workflow + // related tables are stored. + // + // The supported resource type is: + // + // Google BigQuery: + // bigquery.googleapis.com/{dataset} + string dataset = 7; + + // The Cloud Dataflow SDK pipeline options specified by the user. These + // options are passed through the service and are used to recreate the + // SDK pipeline options on the worker in a language agnostic and platform + // independent way. + google.protobuf.Struct sdk_pipeline_options = 8; + + // Experimental settings. + google.protobuf.Any internal_experiments = 9; + + // Identity to run virtual machines as. Defaults to the default account. + string service_account_email = 10; + + // Which Flexible Resource Scheduling mode to run in. + FlexResourceSchedulingGoal flex_resource_scheduling_goal = 11; + + // The Compute Engine region + // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in + // which worker processing should occur, e.g. "us-west1". Mutually exclusive + // with worker_zone. If neither worker_region nor worker_zone is specified, + // default to the control plane's region. + string worker_region = 13; + + // The Compute Engine zone + // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in + // which worker processing should occur, e.g. "us-west1-a". Mutually exclusive + // with worker_region. If neither worker_region nor worker_zone is specified, + // a zone in the control plane's region is chosen based on available capacity. + string worker_zone = 14; + + // Output only. The shuffle mode used for the job. + ShuffleMode shuffle_mode = 15 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Any debugging options to be supplied to the job. + DebugOptions debug_options = 17; +} + +// The packages that must be installed in order for a worker to run the +// steps of the Cloud Dataflow job that will be assigned to its worker +// pool. +// +// This is the mechanism by which the Cloud Dataflow SDK causes code to +// be loaded onto the workers. For example, the Cloud Dataflow Java SDK +// might use this to install jars containing the user's code and all of the +// various dependencies (libraries, data files, etc.) required in order +// for that code to run. +message Package { + // The name of the package. + string name = 1; + + // The resource to read the package from. The supported resource type is: + // + // Google Cloud Storage: + // + // storage.googleapis.com/{bucket} + // bucket.storage.googleapis.com/ + string location = 2; +} + +// Specifies the processing model used by a +// [google.dataflow.v1beta3.Job], which determines the way the Job is +// managed by the Cloud Dataflow service (how workers are scheduled, how +// inputs are sharded, etc). +enum JobType { + // The type of the job is unspecified, or unknown. + JOB_TYPE_UNKNOWN = 0; + + // A batch job with a well-defined end point: data is read, data is + // processed, data is written, and the job is done. + JOB_TYPE_BATCH = 1; + + // A continuously streaming job with no end: data is read, + // processed, and written continuously. + JOB_TYPE_STREAMING = 2; +} + +// Specifies the resource to optimize for in Flexible Resource Scheduling. +enum FlexResourceSchedulingGoal { + // Run in the default mode. + FLEXRS_UNSPECIFIED = 0; + + // Optimize for lower execution time. + FLEXRS_SPEED_OPTIMIZED = 1; + + // Optimize for lower cost. + FLEXRS_COST_OPTIMIZED = 2; +} + +// Describes the data disk used by a workflow job. +message Disk { + // Size of disk in GB. If zero or unspecified, the service will + // attempt to choose a reasonable default. + int32 size_gb = 1; + + // Disk storage type, as defined by Google Compute Engine. This + // must be a disk type appropriate to the project and zone in which + // the workers will run. If unknown or unspecified, the service + // will attempt to choose a reasonable default. + // + // For example, the standard persistent disk type is a resource name + // typically ending in "pd-standard". If SSD persistent disks are + // available, the resource name typically ends with "pd-ssd". The + // actual valid values are defined the Google Compute Engine API, + // not by the Cloud Dataflow API; consult the Google Compute Engine + // documentation for more information about determining the set of + // available disk types for a particular project and zone. + // + // Google Compute Engine Disk types are local to a particular + // project in a particular zone, and so the resource name will + // typically look something like this: + // + // compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd-standard + string disk_type = 2; + + // Directory in a VM where disk is mounted. + string mount_point = 3; +} + +// Provides data to pass through to the worker harness. +message WorkerSettings { + // The base URL for accessing Google Cloud APIs. + // + // When workers access Google Cloud APIs, they logically do so via + // relative URLs. If this field is specified, it supplies the base + // URL to use for resolving these relative URLs. The normative + // algorithm used is defined by RFC 1808, "Relative Uniform Resource + // Locators". + // + // If not specified, the default value is "http://www.googleapis.com/" + string base_url = 1; + + // Whether to send work progress updates to the service. + bool reporting_enabled = 2; + + // The Cloud Dataflow service path relative to the root URL, for example, + // "dataflow/v1b3/projects". + string service_path = 3; + + // The Shuffle service path relative to the root URL, for example, + // "shuffle/v1beta1". + string shuffle_service_path = 4; + + // The ID of the worker running this pipeline. + string worker_id = 5; + + // The prefix of the resources the system should use for temporary + // storage. + // + // The supported resource type is: + // + // Google Cloud Storage: + // + // storage.googleapis.com/{bucket}/{object} + // bucket.storage.googleapis.com/{object} + string temp_storage_prefix = 6; +} + +// Taskrunner configuration settings. +message TaskRunnerSettings { + // The UNIX user ID on the worker VM to use for tasks launched by + // taskrunner; e.g. "root". + string task_user = 1; + + // The UNIX group ID on the worker VM to use for tasks launched by + // taskrunner; e.g. "wheel". + string task_group = 2; + + // The OAuth2 scopes to be requested by the taskrunner in order to + // access the Cloud Dataflow API. + repeated string oauth_scopes = 3; + + // The base URL for the taskrunner to use when accessing Google Cloud APIs. + // + // When workers access Google Cloud APIs, they logically do so via + // relative URLs. If this field is specified, it supplies the base + // URL to use for resolving these relative URLs. The normative + // algorithm used is defined by RFC 1808, "Relative Uniform Resource + // Locators". + // + // If not specified, the default value is "http://www.googleapis.com/" + string base_url = 4; + + // The API version of endpoint, e.g. "v1b3" + string dataflow_api_version = 5; + + // The settings to pass to the parallel worker harness. + WorkerSettings parallel_worker_settings = 6; + + // The location on the worker for task-specific subdirectories. + string base_task_dir = 7; + + // Whether to continue taskrunner if an exception is hit. + bool continue_on_exception = 8; + + // Whether to send taskrunner log info to Google Compute Engine VM serial + // console. + bool log_to_serialconsole = 9; + + // Whether to also send taskrunner log info to stderr. + bool alsologtostderr = 10; + + // Indicates where to put logs. If this is not specified, the logs + // will not be uploaded. + // + // The supported resource type is: + // + // Google Cloud Storage: + // storage.googleapis.com/{bucket}/{object} + // bucket.storage.googleapis.com/{object} + string log_upload_location = 11; + + // The directory on the VM to store logs. + string log_dir = 12; + + // The prefix of the resources the taskrunner should use for + // temporary storage. + // + // The supported resource type is: + // + // Google Cloud Storage: + // storage.googleapis.com/{bucket}/{object} + // bucket.storage.googleapis.com/{object} + string temp_storage_prefix = 13; + + // The command to launch the worker harness. + string harness_command = 14; + + // The file to store the workflow in. + string workflow_file_name = 15; + + // The file to store preprocessing commands in. + string commandlines_file_name = 16; + + // The ID string of the VM. + string vm_id = 17; + + // The suggested backend language. + string language_hint = 18; + + // The streaming worker main class name. + string streaming_worker_main_class = 19; +} + +// Specifies what happens to a resource when a Cloud Dataflow +// [google.dataflow.v1beta3.Job][google.dataflow.v1beta3.Job] has completed. +enum TeardownPolicy { + // The teardown policy isn't specified, or is unknown. + TEARDOWN_POLICY_UNKNOWN = 0; + + // Always teardown the resource. + TEARDOWN_ALWAYS = 1; + + // Teardown the resource on success. This is useful for debugging + // failures. + TEARDOWN_ON_SUCCESS = 2; + + // Never teardown the resource. This is useful for debugging and + // development. + TEARDOWN_NEVER = 3; +} + +// The default set of packages to be staged on a pool of workers. +enum DefaultPackageSet { + // The default set of packages to stage is unknown, or unspecified. + DEFAULT_PACKAGE_SET_UNKNOWN = 0; + + // Indicates that no packages should be staged at the worker unless + // explicitly specified by the job. + DEFAULT_PACKAGE_SET_NONE = 1; + + // Stage packages typically useful to workers written in Java. + DEFAULT_PACKAGE_SET_JAVA = 2; + + // Stage packages typically useful to workers written in Python. + DEFAULT_PACKAGE_SET_PYTHON = 3; +} + +// Specifies the algorithm used to determine the number of worker +// processes to run at any given point in time, based on the amount of +// data left to process, the number of workers, and how quickly +// existing workers are processing data. +enum AutoscalingAlgorithm { + // The algorithm is unknown, or unspecified. + AUTOSCALING_ALGORITHM_UNKNOWN = 0; + + // Disable autoscaling. + AUTOSCALING_ALGORITHM_NONE = 1; + + // Increase worker count over time to reduce job execution time. + AUTOSCALING_ALGORITHM_BASIC = 2; +} + +// Settings for WorkerPool autoscaling. +message AutoscalingSettings { + // The algorithm to use for autoscaling. + AutoscalingAlgorithm algorithm = 1; + + // The maximum number of workers to cap scaling at. + int32 max_num_workers = 2; +} + +// Specifies how IP addresses should be allocated to the worker machines. +enum WorkerIPAddressConfiguration { + // The configuration is unknown, or unspecified. + WORKER_IP_UNSPECIFIED = 0; + + // Workers should have public IP addresses. + WORKER_IP_PUBLIC = 1; + + // Workers should have private IP addresses. + WORKER_IP_PRIVATE = 2; +} + +// Defines a SDK harness container for executing Dataflow pipelines. +message SdkHarnessContainerImage { + // A docker container image that resides in Google Container Registry. + string container_image = 1; + + // If true, recommends the Dataflow service to use only one core per SDK + // container instance with this image. If false (or unset) recommends using + // more than one core per SDK container instance with this image for + // efficiency. Note that Dataflow service may choose to override this property + // if needed. + bool use_single_core_per_container = 2; + + // Environment ID for the Beam runner API proto Environment that corresponds + // to the current SDK Harness. + string environment_id = 3; + + // The set of capabilities enumerated in the above Environment proto. See also + // https://github.com/apache/beam/blob/master/model/pipeline/src/main/proto/beam_runner_api.proto + repeated string capabilities = 4; +} + +// Describes one particular pool of Cloud Dataflow workers to be +// instantiated by the Cloud Dataflow service in order to perform the +// computations required by a job. Note that a workflow job may use +// multiple pools, in order to match the various computational +// requirements of the various stages of the job. +message WorkerPool { + // The kind of the worker pool; currently only `harness` and `shuffle` + // are supported. + string kind = 1; + + // Number of Google Compute Engine workers in this pool needed to + // execute the job. If zero or unspecified, the service will + // attempt to choose a reasonable default. + int32 num_workers = 2; + + // Packages to be installed on workers. + repeated Package packages = 3; + + // The default package set to install. This allows the service to + // select a default set of packages which are useful to worker + // harnesses written in a particular language. + DefaultPackageSet default_package_set = 4; + + // Machine type (e.g. "n1-standard-1"). If empty or unspecified, the + // service will attempt to choose a reasonable default. + string machine_type = 5; + + // Sets the policy for determining when to turndown worker pool. + // Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and + // `TEARDOWN_NEVER`. + // `TEARDOWN_ALWAYS` means workers are always torn down regardless of whether + // the job succeeds. `TEARDOWN_ON_SUCCESS` means workers are torn down + // if the job succeeds. `TEARDOWN_NEVER` means the workers are never torn + // down. + // + // If the workers are not torn down by the service, they will + // continue to run and use Google Compute Engine VM resources in the + // user's project until they are explicitly terminated by the user. + // Because of this, Google recommends using the `TEARDOWN_ALWAYS` + // policy except for small, manually supervised test jobs. + // + // If unknown or unspecified, the service will attempt to choose a reasonable + // default. + TeardownPolicy teardown_policy = 6; + + // Size of root disk for VMs, in GB. If zero or unspecified, the service will + // attempt to choose a reasonable default. + int32 disk_size_gb = 7; + + // Type of root disk for VMs. If empty or unspecified, the service will + // attempt to choose a reasonable default. + string disk_type = 16; + + // Fully qualified source image for disks. + string disk_source_image = 8; + + // Zone to run the worker pools in. If empty or unspecified, the service + // will attempt to choose a reasonable default. + string zone = 9; + + // Settings passed through to Google Compute Engine workers when + // using the standard Dataflow task runner. Users should ignore + // this field. + TaskRunnerSettings taskrunner_settings = 10; + + // The action to take on host maintenance, as defined by the Google + // Compute Engine API. + string on_host_maintenance = 11; + + // Data disks that are used by a VM in this workflow. + repeated Disk data_disks = 12; + + // Metadata to set on the Google Compute Engine VMs. + map<string, string> metadata = 13; + + // Settings for autoscaling of this WorkerPool. + AutoscalingSettings autoscaling_settings = 14; + + // Extra arguments for this worker pool. + google.protobuf.Any pool_args = 15; + + // Network to which VMs will be assigned. If empty or unspecified, + // the service will use the network "default". + string network = 17; + + // Subnetwork to which VMs will be assigned, if desired. Expected to be of + // the form "regions/REGION/subnetworks/SUBNETWORK". + string subnetwork = 19; + + // Required. Docker container image that executes the Cloud Dataflow worker + // harness, residing in Google Container Registry. + // + // Deprecated for the Fn API path. Use sdk_harness_container_images instead. + string worker_harness_container_image = 18; + + // The number of threads per worker harness. If empty or unspecified, the + // service will choose a number of threads (according to the number of cores + // on the selected machine type for batch, or 1 by convention for streaming). + int32 num_threads_per_worker = 20; + + // Configuration for VM IPs. + WorkerIPAddressConfiguration ip_configuration = 21; + + // Set of SDK harness containers needed to execute this pipeline. This will + // only be set in the Fn API path. For non-cross-language pipelines this + // should have only one entry. Cross-language pipelines will have two or more + // entries. + repeated SdkHarnessContainerImage sdk_harness_container_images = 22; +} + +// Specifies the shuffle mode used by a +// [google.dataflow.v1beta3.Job], which determines the approach data is shuffled +// during processing. More details in: +// https://cloud.google.com/dataflow/docs/guides/deploying-a-pipeline#dataflow-shuffle +enum ShuffleMode { + // Shuffle mode information is not available. + SHUFFLE_MODE_UNSPECIFIED = 0; + + // Shuffle is done on the worker VMs. + VM_BASED = 1; + + // Shuffle is done on the service side. + SERVICE_BASED = 2; +} + +// Describes any options that have an effect on the debugging of pipelines. +message DebugOptions { + // When true, enables the logging of the literal hot key to the user's Cloud + // Logging. + bool enable_hot_key_logging = 1; +} diff --git a/third_party/googleapis/google/dataflow/v1beta3/jobs.proto b/third_party/googleapis/google/dataflow/v1beta3/jobs.proto new file mode 100644 index 0000000..359b787 --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/jobs.proto @@ -0,0 +1,926 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.dataflow.v1beta3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/dataflow/v1beta3/environment.proto"; +import "google/dataflow/v1beta3/snapshots.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3"; +option go_package = "google.golang.org/genproto/googleapis/dataflow/v1beta3;dataflow"; +option java_multiple_files = true; +option java_outer_classname = "JobsProto"; +option java_package = "com.google.dataflow.v1beta3"; +option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3"; +option ruby_package = "Google::Cloud::Dataflow::V1beta3"; + +// Provides a method to create and modify Google Cloud Dataflow jobs. +// A Job is a multi-stage computation graph run by the Cloud Dataflow service. +service JobsV1Beta3 { + option (google.api.default_host) = "dataflow.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/compute," + "https://www.googleapis.com/auth/compute.readonly," + "https://www.googleapis.com/auth/userinfo.email"; + + // Creates a Cloud Dataflow job. + // + // To create a job, we recommend using `projects.locations.jobs.create` with a + // [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using + // `projects.jobs.create` is not recommended, as your job will always start + // in `us-central1`. + rpc CreateJob(CreateJobRequest) returns (Job) { + option (google.api.http) = { + post: "/v1b3/projects/{project_id}/locations/{location}/jobs" + body: "job" + additional_bindings { + post: "/v1b3/projects/{project_id}/jobs" + body: "job" + } + }; + } + + // Gets the state of the specified Cloud Dataflow job. + // + // To get the state of a job, we recommend using `projects.locations.jobs.get` + // with a [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using + // `projects.jobs.get` is not recommended, as you can only get the state of + // jobs that are running in `us-central1`. + rpc GetJob(GetJobRequest) returns (Job) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}" + additional_bindings { + get: "/v1b3/projects/{project_id}/jobs/{job_id}" + } + }; + } + + // Updates the state of an existing Cloud Dataflow job. + // + // To update the state of an existing job, we recommend using + // `projects.locations.jobs.update` with a [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using + // `projects.jobs.update` is not recommended, as you can only update the state + // of jobs that are running in `us-central1`. + rpc UpdateJob(UpdateJobRequest) returns (Job) { + option (google.api.http) = { + put: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}" + body: "job" + additional_bindings { + put: "/v1b3/projects/{project_id}/jobs/{job_id}" + body: "job" + } + }; + } + + // List the jobs of a project. + // + // To list the jobs of a project in a region, we recommend using + // `projects.locations.jobs.list` with a [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). To + // list the all jobs across all regions, use `projects.jobs.aggregated`. Using + // `projects.jobs.list` is not recommended, as you can only get the list of + // jobs that are running in `us-central1`. + rpc ListJobs(ListJobsRequest) returns (ListJobsResponse) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/jobs" + additional_bindings { + get: "/v1b3/projects/{project_id}/jobs" + } + }; + } + + // List the jobs of a project across all regions. + rpc AggregatedListJobs(ListJobsRequest) returns (ListJobsResponse) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/jobs:aggregated" + }; + } + + // Check for existence of active jobs in the given project across all regions. + rpc CheckActiveJobs(CheckActiveJobsRequest) returns (CheckActiveJobsResponse) { + } + + // Snapshot the state of a streaming job. + rpc SnapshotJob(SnapshotJobRequest) returns (Snapshot) { + option (google.api.http) = { + post: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}:snapshot" + body: "*" + additional_bindings { + post: "/v1b3/projects/{project_id}/jobs/{job_id}:snapshot" + body: "*" + } + }; + } +} + +// Defines a job to be run by the Cloud Dataflow service. +message Job { + // The unique ID of this job. + // + // This field is set by the Cloud Dataflow service when the Job is + // created, and is immutable for the life of the job. + string id = 1; + + // The ID of the Cloud Platform project that the job belongs to. + string project_id = 2; + + // The user-specified Cloud Dataflow job name. + // + // Only one Job with a given name may exist in a project at any + // given time. If a caller attempts to create a Job with the same + // name as an already-existing Job, the attempt returns the + // existing Job. + // + // The name must match the regular expression + // `[a-z]([-a-z0-9]{0,1022}[a-z0-9])?` + string name = 3; + + // The type of Cloud Dataflow job. + JobType type = 4; + + // The environment for the job. + Environment environment = 5; + + // Exactly one of step or steps_location should be specified. + // + // The top-level steps that constitute the entire job. Only retrieved with + // JOB_VIEW_ALL. + repeated Step steps = 6; + + // The Cloud Storage location where the steps are stored. + string steps_location = 24; + + // The current state of the job. + // + // Jobs are created in the `JOB_STATE_STOPPED` state unless otherwise + // specified. + // + // A job in the `JOB_STATE_RUNNING` state may asynchronously enter a + // terminal state. After a job has reached a terminal state, no + // further state updates may be made. + // + // This field may be mutated by the Cloud Dataflow service; + // callers cannot mutate it. + JobState current_state = 7; + + // The timestamp associated with the current state. + google.protobuf.Timestamp current_state_time = 8; + + // The job's requested state. + // + // `UpdateJob` may be used to switch between the `JOB_STATE_STOPPED` and + // `JOB_STATE_RUNNING` states, by setting requested_state. `UpdateJob` may + // also be used to directly set a job's requested state to + // `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably terminating the + // job if it has not already reached a terminal state. + JobState requested_state = 9; + + // Deprecated. + JobExecutionInfo execution_info = 10; + + // The timestamp when the job was initially created. Immutable and set by the + // Cloud Dataflow service. + google.protobuf.Timestamp create_time = 11; + + // If this job is an update of an existing job, this field is the job ID + // of the job it replaced. + // + // When sending a `CreateJobRequest`, you can update a job by specifying it + // here. The job named here is stopped, and its intermediate state is + // transferred to this job. + string replace_job_id = 12; + + // The map of transform name prefixes of the job to be replaced to the + // corresponding name prefixes of the new job. + map<string, string> transform_name_mapping = 13; + + // The client's unique identifier of the job, re-used across retried attempts. + // If this field is set, the service will ensure its uniqueness. + // The request to create a job will fail if the service has knowledge of a + // previously submitted job with the same client's ID and job name. + // The caller may use this field to ensure idempotence of job + // creation across retried attempts to create a job. + // By default, the field is empty and, in that case, the service ignores it. + string client_request_id = 14; + + // If another job is an update of this job (and thus, this job is in + // `JOB_STATE_UPDATED`), this field contains the ID of that job. + string replaced_by_job_id = 15; + + // A set of files the system should be aware of that are used + // for temporary storage. These temporary files will be + // removed on job completion. + // No duplicates are allowed. + // No file patterns are supported. + // + // The supported files are: + // + // Google Cloud Storage: + // + // storage.googleapis.com/{bucket}/{object} + // bucket.storage.googleapis.com/{object} + repeated string temp_files = 16; + + // User-defined labels for this job. + // + // The labels map can contain no more than 64 entries. Entries of the labels + // map are UTF8 strings that comply with the following restrictions: + // + // * Keys must conform to regexp: [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62} + // * Values must conform to regexp: [\p{Ll}\p{Lo}\p{N}_-]{0,63} + // * Both keys and values are additionally constrained to be <= 128 bytes in + // size. + map<string, string> labels = 17; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains this job. + string location = 18; + + // Preliminary field: The format of this data may change at any time. + // A description of the user pipeline and stages through which it is executed. + // Created by Cloud Dataflow service. Only retrieved with + // JOB_VIEW_DESCRIPTION or JOB_VIEW_ALL. + PipelineDescription pipeline_description = 19; + + // This field may be mutated by the Cloud Dataflow service; + // callers cannot mutate it. + repeated ExecutionStageState stage_states = 20; + + // This field is populated by the Dataflow service to support filtering jobs + // by the metadata values provided here. Populated for ListJobs and all GetJob + // views SUMMARY and higher. + JobMetadata job_metadata = 21; + + // The timestamp when the job was started (transitioned to JOB_STATE_PENDING). + // Flexible resource scheduling jobs are started with some delay after job + // creation, so start_time is unset before start and is updated when the + // job is started by the Cloud Dataflow service. For other jobs, start_time + // always equals to create_time and is immutable and set by the Cloud Dataflow + // service. + google.protobuf.Timestamp start_time = 22; + + // If this is specified, the job's initial state is populated from the given + // snapshot. + string created_from_snapshot_id = 23; + + // Reserved for future use. This field is set only in responses from the + // server; it is ignored if it is set in any requests. + bool satisfies_pzs = 25; +} + +// Metadata for a Datastore connector used by the job. +message DatastoreIODetails { + // Namespace used in the connection. + string namespace = 1; + + // ProjectId accessed in the connection. + string project_id = 2; +} + +// Metadata for a Pub/Sub connector used by the job. +message PubSubIODetails { + // Topic accessed in the connection. + string topic = 1; + + // Subscription used in the connection. + string subscription = 2; +} + +// Metadata for a File connector used by the job. +message FileIODetails { + // File Pattern used to access files by the connector. + string file_pattern = 1; +} + +// Metadata for a Cloud Bigtable connector used by the job. +message BigTableIODetails { + // ProjectId accessed in the connection. + string project_id = 1; + + // InstanceId accessed in the connection. + string instance_id = 2; + + // TableId accessed in the connection. + string table_id = 3; +} + +// Metadata for a BigQuery connector used by the job. +message BigQueryIODetails { + // Table accessed in the connection. + string table = 1; + + // Dataset accessed in the connection. + string dataset = 2; + + // Project accessed in the connection. + string project_id = 3; + + // Query used to access data in the connection. + string query = 4; +} + +// Metadata for a Spanner connector used by the job. +message SpannerIODetails { + // ProjectId accessed in the connection. + string project_id = 1; + + // InstanceId accessed in the connection. + string instance_id = 2; + + // DatabaseId accessed in the connection. + string database_id = 3; +} + +// The version of the SDK used to run the job. +message SdkVersion { + // The support status of the SDK used to run the job. + enum SdkSupportStatus { + // Cloud Dataflow is unaware of this version. + UNKNOWN = 0; + + // This is a known version of an SDK, and is supported. + SUPPORTED = 1; + + // A newer version of the SDK family exists, and an update is recommended. + STALE = 2; + + // This version of the SDK is deprecated and will eventually be + // unsupported. + DEPRECATED = 3; + + // Support for this SDK version has ended and it should no longer be used. + UNSUPPORTED = 4; + } + + // The version of the SDK used to run the job. + string version = 1; + + // A readable string describing the version of the SDK. + string version_display_name = 2; + + // The support status for this SDK version. + SdkSupportStatus sdk_support_status = 3; +} + +// Metadata available primarily for filtering jobs. Will be included in the +// ListJob response and Job SUMMARY view. +message JobMetadata { + // The SDK version used to run the job. + SdkVersion sdk_version = 1; + + // Identification of a Spanner source used in the Dataflow job. + repeated SpannerIODetails spanner_details = 2; + + // Identification of a BigQuery source used in the Dataflow job. + repeated BigQueryIODetails bigquery_details = 3; + + // Identification of a Cloud Bigtable source used in the Dataflow job. + repeated BigTableIODetails big_table_details = 4; + + // Identification of a Pub/Sub source used in the Dataflow job. + repeated PubSubIODetails pubsub_details = 5; + + // Identification of a File source used in the Dataflow job. + repeated FileIODetails file_details = 6; + + // Identification of a Datastore source used in the Dataflow job. + repeated DatastoreIODetails datastore_details = 7; +} + +// A message describing the state of a particular execution stage. +message ExecutionStageState { + // The name of the execution stage. + string execution_stage_name = 1; + + // Executions stage states allow the same set of values as JobState. + JobState execution_stage_state = 2; + + // The time at which the stage transitioned to this state. + google.protobuf.Timestamp current_state_time = 3; +} + +// A descriptive representation of submitted pipeline as well as the executed +// form. This data is provided by the Dataflow service for ease of visualizing +// the pipeline and interpreting Dataflow provided metrics. +message PipelineDescription { + // Description of each transform in the pipeline and collections between them. + repeated TransformSummary original_pipeline_transform = 1; + + // Description of each stage of execution of the pipeline. + repeated ExecutionStageSummary execution_pipeline_stage = 2; + + // Pipeline level display data. + repeated DisplayData display_data = 3; +} + +// Type of transform or stage operation. +enum KindType { + // Unrecognized transform type. + UNKNOWN_KIND = 0; + + // ParDo transform. + PAR_DO_KIND = 1; + + // Group By Key transform. + GROUP_BY_KEY_KIND = 2; + + // Flatten transform. + FLATTEN_KIND = 3; + + // Read transform. + READ_KIND = 4; + + // Write transform. + WRITE_KIND = 5; + + // Constructs from a constant value, such as with Create.of. + CONSTANT_KIND = 6; + + // Creates a Singleton view of a collection. + SINGLETON_KIND = 7; + + // Opening or closing a shuffle session, often as part of a GroupByKey. + SHUFFLE_KIND = 8; +} + +// Description of the type, names/ids, and input/outputs for a transform. +message TransformSummary { + // Type of transform. + KindType kind = 1; + + // SDK generated id of this transform instance. + string id = 2; + + // User provided name for this transform instance. + string name = 3; + + // Transform-specific display data. + repeated DisplayData display_data = 4; + + // User names for all collection outputs to this transform. + repeated string output_collection_name = 5; + + // User names for all collection inputs to this transform. + repeated string input_collection_name = 6; +} + +// Description of the composing transforms, names/ids, and input/outputs of a +// stage of execution. Some composing transforms and sources may have been +// generated by the Dataflow service during execution planning. +message ExecutionStageSummary { + // Description of an input or output of an execution stage. + message StageSource { + // Human-readable name for this source; may be user or system generated. + string user_name = 1; + + // Dataflow service generated name for this source. + string name = 2; + + // User name for the original user transform or collection with which this + // source is most closely associated. + string original_transform_or_collection = 3; + + // Size of the source, if measurable. + int64 size_bytes = 4; + } + + // Description of a transform executed as part of an execution stage. + message ComponentTransform { + // Human-readable name for this transform; may be user or system generated. + string user_name = 1; + + // Dataflow service generated name for this source. + string name = 2; + + // User name for the original user transform with which this transform is + // most closely associated. + string original_transform = 3; + } + + // Description of an interstitial value between transforms in an execution + // stage. + message ComponentSource { + // Human-readable name for this transform; may be user or system generated. + string user_name = 1; + + // Dataflow service generated name for this source. + string name = 2; + + // User name for the original user transform or collection with which this + // source is most closely associated. + string original_transform_or_collection = 3; + } + + // Dataflow service generated name for this stage. + string name = 1; + + // Dataflow service generated id for this stage. + string id = 2; + + // Type of transform this stage is executing. + KindType kind = 3; + + // Input sources for this stage. + repeated StageSource input_source = 4; + + // Output sources for this stage. + repeated StageSource output_source = 5; + + // Other stages that must complete before this stage can run. + repeated string prerequisite_stage = 8; + + // Transforms that comprise this execution stage. + repeated ComponentTransform component_transform = 6; + + // Collections produced and consumed by component transforms of this stage. + repeated ComponentSource component_source = 7; +} + +// Data provided with a pipeline or transform to provide descriptive info. +message DisplayData { + // The key identifying the display data. + // This is intended to be used as a label for the display data + // when viewed in a dax monitoring system. + string key = 1; + + // The namespace for the key. This is usually a class name or programming + // language namespace (i.e. python module) which defines the display data. + // This allows a dax monitoring system to specially handle the data + // and perform custom rendering. + string namespace = 2; + + // Various value types which can be used for display data. Only one will be + // set. + oneof Value { + // Contains value if the data is of string type. + string str_value = 4; + + // Contains value if the data is of int64 type. + int64 int64_value = 5; + + // Contains value if the data is of float type. + float float_value = 6; + + // Contains value if the data is of java class type. + string java_class_value = 7; + + // Contains value if the data is of timestamp type. + google.protobuf.Timestamp timestamp_value = 8; + + // Contains value if the data is of duration type. + google.protobuf.Duration duration_value = 9; + + // Contains value if the data is of a boolean type. + bool bool_value = 10; + } + + // A possible additional shorter value to display. + // For example a java_class_name_value of com.mypackage.MyDoFn + // will be stored with MyDoFn as the short_str_value and + // com.mypackage.MyDoFn as the java_class_name value. + // short_str_value can be displayed and java_class_name_value + // will be displayed as a tooltip. + string short_str_value = 11; + + // An optional full URL. + string url = 12; + + // An optional label to display in a dax UI for the element. + string label = 13; +} + +// Defines a particular step within a Cloud Dataflow job. +// +// A job consists of multiple steps, each of which performs some +// specific operation as part of the overall job. Data is typically +// passed from one step to another as part of the job. +// +// Here's an example of a sequence of steps which together implement a +// Map-Reduce job: +// +// * Read a collection of data from some source, parsing the +// collection's elements. +// +// * Validate the elements. +// +// * Apply a user-defined function to map each element to some value +// and extract an element-specific key value. +// +// * Group elements with the same key into a single element with +// that key, transforming a multiply-keyed collection into a +// uniquely-keyed collection. +// +// * Write the elements out to some data sink. +// +// Note that the Cloud Dataflow service may be used to run many different +// types of jobs, not just Map-Reduce. +message Step { + // The kind of step in the Cloud Dataflow job. + string kind = 1; + + // The name that identifies the step. This must be unique for each + // step with respect to all other steps in the Cloud Dataflow job. + string name = 2; + + // Named properties associated with the step. Each kind of + // predefined step has its own required set of properties. + // Must be provided on Create. Only retrieved with JOB_VIEW_ALL. + google.protobuf.Struct properties = 3; +} + +// Describes the overall state of a [google.dataflow.v1beta3.Job][google.dataflow.v1beta3.Job]. +enum JobState { + // The job's run state isn't specified. + JOB_STATE_UNKNOWN = 0; + + // `JOB_STATE_STOPPED` indicates that the job has not + // yet started to run. + JOB_STATE_STOPPED = 1; + + // `JOB_STATE_RUNNING` indicates that the job is currently running. + JOB_STATE_RUNNING = 2; + + // `JOB_STATE_DONE` indicates that the job has successfully completed. + // This is a terminal job state. This state may be set by the Cloud Dataflow + // service, as a transition from `JOB_STATE_RUNNING`. It may also be set via a + // Cloud Dataflow `UpdateJob` call, if the job has not yet reached a terminal + // state. + JOB_STATE_DONE = 3; + + // `JOB_STATE_FAILED` indicates that the job has failed. This is a + // terminal job state. This state may only be set by the Cloud Dataflow + // service, and only as a transition from `JOB_STATE_RUNNING`. + JOB_STATE_FAILED = 4; + + // `JOB_STATE_CANCELLED` indicates that the job has been explicitly + // cancelled. This is a terminal job state. This state may only be + // set via a Cloud Dataflow `UpdateJob` call, and only if the job has not + // yet reached another terminal state. + JOB_STATE_CANCELLED = 5; + + // `JOB_STATE_UPDATED` indicates that the job was successfully updated, + // meaning that this job was stopped and another job was started, inheriting + // state from this one. This is a terminal job state. This state may only be + // set by the Cloud Dataflow service, and only as a transition from + // `JOB_STATE_RUNNING`. + JOB_STATE_UPDATED = 6; + + // `JOB_STATE_DRAINING` indicates that the job is in the process of draining. + // A draining job has stopped pulling from its input sources and is processing + // any data that remains in-flight. This state may be set via a Cloud Dataflow + // `UpdateJob` call, but only as a transition from `JOB_STATE_RUNNING`. Jobs + // that are draining may only transition to `JOB_STATE_DRAINED`, + // `JOB_STATE_CANCELLED`, or `JOB_STATE_FAILED`. + JOB_STATE_DRAINING = 7; + + // `JOB_STATE_DRAINED` indicates that the job has been drained. + // A drained job terminated by stopping pulling from its input sources and + // processing any data that remained in-flight when draining was requested. + // This state is a terminal state, may only be set by the Cloud Dataflow + // service, and only as a transition from `JOB_STATE_DRAINING`. + JOB_STATE_DRAINED = 8; + + // `JOB_STATE_PENDING` indicates that the job has been created but is not yet + // running. Jobs that are pending may only transition to `JOB_STATE_RUNNING`, + // or `JOB_STATE_FAILED`. + JOB_STATE_PENDING = 9; + + // `JOB_STATE_CANCELLING` indicates that the job has been explicitly cancelled + // and is in the process of stopping. Jobs that are cancelling may only + // transition to `JOB_STATE_CANCELLED` or `JOB_STATE_FAILED`. + JOB_STATE_CANCELLING = 10; + + // `JOB_STATE_QUEUED` indicates that the job has been created but is being + // delayed until launch. Jobs that are queued may only transition to + // `JOB_STATE_PENDING` or `JOB_STATE_CANCELLED`. + JOB_STATE_QUEUED = 11; + + // `JOB_STATE_RESOURCE_CLEANING_UP` indicates that the batch job's associated + // resources are currently being cleaned up after a successful run. + // Currently, this is an opt-in feature, please reach out to Cloud support + // team if you are interested. + JOB_STATE_RESOURCE_CLEANING_UP = 12; +} + +// Additional information about how a Cloud Dataflow job will be executed that +// isn't contained in the submitted job. +message JobExecutionInfo { + // A mapping from each stage to the information about that stage. + map<string, JobExecutionStageInfo> stages = 1; +} + +// Contains information about how a particular +// [google.dataflow.v1beta3.Step][google.dataflow.v1beta3.Step] will be executed. +message JobExecutionStageInfo { + // The steps associated with the execution stage. + // Note that stages may have several steps, and that a given step + // might be run by more than one stage. + repeated string step_name = 1; +} + +// Selector for how much information is returned in Job responses. +enum JobView { + // The job view to return isn't specified, or is unknown. + // Responses will contain at least the `JOB_VIEW_SUMMARY` information, + // and may contain additional information. + JOB_VIEW_UNKNOWN = 0; + + // Request summary information only: + // Project ID, Job ID, job name, job type, job status, start/end time, + // and Cloud SDK version details. + JOB_VIEW_SUMMARY = 1; + + // Request all information available for this job. + JOB_VIEW_ALL = 2; + + // Request summary info and limited job description data for steps, labels and + // environment. + JOB_VIEW_DESCRIPTION = 3; +} + +// Request to create a Cloud Dataflow job. +message CreateJobRequest { + // The ID of the Cloud Platform project that the job belongs to. + string project_id = 1; + + // The job to create. + Job job = 2; + + // The level of information requested in response. + JobView view = 3; + + // Deprecated. This field is now in the Job message. + string replace_job_id = 4; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains this job. + string location = 5; +} + +// Request to get the state of a Cloud Dataflow job. +message GetJobRequest { + // The ID of the Cloud Platform project that the job belongs to. + string project_id = 1; + + // The job ID. + string job_id = 2; + + // The level of information requested in response. + JobView view = 3; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains this job. + string location = 4; +} + +// Request to update a Cloud Dataflow job. +message UpdateJobRequest { + // The ID of the Cloud Platform project that the job belongs to. + string project_id = 1; + + // The job ID. + string job_id = 2; + + // The updated job. + // Only the job state is updatable; other fields will be ignored. + Job job = 3; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains this job. + string location = 4; +} + +// Request to list Cloud Dataflow jobs. +message ListJobsRequest { + // This field filters out and returns jobs in the specified job state. The + // order of data returned is determined by the filter used, and is subject to + // change. + enum Filter { + // The filter isn't specified, or is unknown. This returns all jobs ordered + // on descending `JobUuid`. + UNKNOWN = 0; + + // Returns all running jobs first ordered on creation timestamp, then + // returns all terminated jobs ordered on the termination timestamp. + ALL = 1; + + // Filters the jobs that have a terminated state, ordered on the + // termination timestamp. Example terminated states: `JOB_STATE_STOPPED`, + // `JOB_STATE_UPDATED`, `JOB_STATE_DRAINED`, etc. + TERMINATED = 2; + + // Filters the jobs that are running ordered on the creation timestamp. + ACTIVE = 3; + } + + // The kind of filter to use. + Filter filter = 5; + + // The project which owns the jobs. + string project_id = 1; + + // Deprecated. ListJobs always returns summaries now. + // Use GetJob for other JobViews. + JobView view = 2 [deprecated = true]; + + // If there are many jobs, limit response to at most this many. + // The actual number of jobs returned will be the lesser of max_responses + // and an unspecified server-defined limit. + int32 page_size = 3; + + // Set this to the 'next_page_token' field of a previous response + // to request additional results in a long list. + string page_token = 4; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains this job. + string location = 17; +} + +// Indicates which [regional endpoint] +// (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) failed +// to respond to a request for data. +message FailedLocation { + // The name of the [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // failed to respond. + string name = 1; +} + +// Response to a request to list Cloud Dataflow jobs in a project. This might +// be a partial response, depending on the page size in the ListJobsRequest. +// However, if the project does not have any jobs, an instance of +// ListJobsResponse is not returned and the requests's response +// body is empty {}. +message ListJobsResponse { + // A subset of the requested job information. + repeated Job jobs = 1; + + // Set if there may be more results than fit in this response. + string next_page_token = 2; + + // Zero or more messages describing the [regional endpoints] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // failed to respond. + repeated FailedLocation failed_location = 3; +} + +// Request to create a snapshot of a job. +message SnapshotJobRequest { + // The project which owns the job to be snapshotted. + string project_id = 1; + + // The job to be snapshotted. + string job_id = 2; + + // TTL for the snapshot. + google.protobuf.Duration ttl = 3; + + // The location that contains this job. + string location = 4; + + // If true, perform snapshots for sources which support this. + bool snapshot_sources = 5; + + // User specified description of the snapshot. Maybe empty. + string description = 6; +} + +// Request to check is active jobs exists for a project +message CheckActiveJobsRequest { + // The project which owns the jobs. + string project_id = 1; +} + +// Response for CheckActiveJobsRequest. +message CheckActiveJobsResponse { + // If True, active jobs exists for project. False otherwise. + bool active_jobs_exist = 1; +} diff --git a/third_party/googleapis/google/dataflow/v1beta3/messages.proto b/third_party/googleapis/google/dataflow/v1beta3/messages.proto new file mode 100644 index 0000000..f8d151e --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/messages.proto @@ -0,0 +1,238 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.dataflow.v1beta3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3"; +option go_package = "google.golang.org/genproto/googleapis/dataflow/v1beta3;dataflow"; +option java_multiple_files = true; +option java_outer_classname = "MessagesProto"; +option java_package = "com.google.dataflow.v1beta3"; +option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3"; +option ruby_package = "Google::Cloud::Dataflow::V1beta3"; + +// The Dataflow Messages API is used for monitoring the progress of +// Dataflow jobs. +service MessagesV1Beta3 { + option (google.api.default_host) = "dataflow.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/compute," + "https://www.googleapis.com/auth/compute.readonly," + "https://www.googleapis.com/auth/userinfo.email"; + + // Request the job status. + // + // To request the status of a job, we recommend using + // `projects.locations.jobs.messages.list` with a [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using + // `projects.jobs.messages.list` is not recommended, as you can only request + // the status of jobs that are running in `us-central1`. + rpc ListJobMessages(ListJobMessagesRequest) returns (ListJobMessagesResponse) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}/messages" + additional_bindings { + get: "/v1b3/projects/{project_id}/jobs/{job_id}/messages" + } + }; + } +} + +// A particular message pertaining to a Dataflow job. +message JobMessage { + // Deprecated. + string id = 1; + + // The timestamp of the message. + google.protobuf.Timestamp time = 2; + + // The text of the message. + string message_text = 3; + + // Importance level of the message. + JobMessageImportance message_importance = 4; +} + +// Indicates the importance of the message. +enum JobMessageImportance { + // The message importance isn't specified, or is unknown. + JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0; + + // The message is at the 'debug' level: typically only useful for + // software engineers working on the code the job is running. + // Typically, Dataflow pipeline runners do not display log messages + // at this level by default. + JOB_MESSAGE_DEBUG = 1; + + // The message is at the 'detailed' level: somewhat verbose, but + // potentially useful to users. Typically, Dataflow pipeline + // runners do not display log messages at this level by default. + // These messages are displayed by default in the Dataflow + // monitoring UI. + JOB_MESSAGE_DETAILED = 2; + + // The message is at the 'basic' level: useful for keeping + // track of the execution of a Dataflow pipeline. Typically, + // Dataflow pipeline runners display log messages at this level by + // default, and these messages are displayed by default in the + // Dataflow monitoring UI. + JOB_MESSAGE_BASIC = 5; + + // The message is at the 'warning' level: indicating a condition + // pertaining to a job which may require human intervention. + // Typically, Dataflow pipeline runners display log messages at this + // level by default, and these messages are displayed by default in + // the Dataflow monitoring UI. + JOB_MESSAGE_WARNING = 3; + + // The message is at the 'error' level: indicating a condition + // preventing a job from succeeding. Typically, Dataflow pipeline + // runners display log messages at this level by default, and these + // messages are displayed by default in the Dataflow monitoring UI. + JOB_MESSAGE_ERROR = 4; +} + +// A rich message format, including a human readable string, a key for +// identifying the message, and structured data associated with the message for +// programmatic consumption. +message StructuredMessage { + // Structured data associated with this message. + message Parameter { + // Key or name for this parameter. + string key = 1; + + // Value for this parameter. + google.protobuf.Value value = 2; + } + + // Human-readable version of message. + string message_text = 1; + + // Identifier for this message type. Used by external systems to + // internationalize or personalize message. + string message_key = 2; + + // The structured data associated with this message. + repeated Parameter parameters = 3; +} + +// A structured message reporting an autoscaling decision made by the Dataflow +// service. +message AutoscalingEvent { + // Indicates the type of autoscaling event. + enum AutoscalingEventType { + // Default type for the enum. Value should never be returned. + TYPE_UNKNOWN = 0; + + // The TARGET_NUM_WORKERS_CHANGED type should be used when the target + // worker pool size has changed at the start of an actuation. An event + // should always be specified as TARGET_NUM_WORKERS_CHANGED if it reflects + // a change in the target_num_workers. + TARGET_NUM_WORKERS_CHANGED = 1; + + // The CURRENT_NUM_WORKERS_CHANGED type should be used when actual worker + // pool size has been changed, but the target_num_workers has not changed. + CURRENT_NUM_WORKERS_CHANGED = 2; + + // The ACTUATION_FAILURE type should be used when we want to report + // an error to the user indicating why the current number of workers + // in the pool could not be changed. + // Displayed in the current status and history widgets. + ACTUATION_FAILURE = 3; + + // Used when we want to report to the user a reason why we are + // not currently adjusting the number of workers. + // Should specify both target_num_workers, current_num_workers and a + // decision_message. + NO_CHANGE = 4; + } + + // The current number of workers the job has. + int64 current_num_workers = 1; + + // The target number of workers the worker pool wants to resize to use. + int64 target_num_workers = 2; + + // The type of autoscaling event to report. + AutoscalingEventType event_type = 3; + + // A message describing why the system decided to adjust the current + // number of workers, why it failed, or why the system decided to + // not make any changes to the number of workers. + StructuredMessage description = 4; + + // The time this event was emitted to indicate a new target or current + // num_workers value. + google.protobuf.Timestamp time = 5; + + // A short and friendly name for the worker pool this event refers to. + string worker_pool = 7; +} + +// Request to list job messages. +// Up to max_results messages will be returned in the time range specified +// starting with the oldest messages first. If no time range is specified +// the results with start with the oldest message. +message ListJobMessagesRequest { + // A project id. + string project_id = 1; + + // The job to get messages about. + string job_id = 2; + + // Filter to only get messages with importance >= level + JobMessageImportance minimum_importance = 3; + + // If specified, determines the maximum number of messages to + // return. If unspecified, the service may choose an appropriate + // default, or may return an arbitrarily large number of results. + int32 page_size = 4; + + // If supplied, this should be the value of next_page_token returned + // by an earlier call. This will cause the next page of results to + // be returned. + string page_token = 5; + + // If specified, return only messages with timestamps >= start_time. + // The default is the job creation time (i.e. beginning of messages). + google.protobuf.Timestamp start_time = 6; + + // Return only messages with timestamps < end_time. The default is now + // (i.e. return up to the latest messages available). + google.protobuf.Timestamp end_time = 7; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains the job specified by job_id. + string location = 8; +} + +// Response to a request to list job messages. +message ListJobMessagesResponse { + // Messages in ascending timestamp order. + repeated JobMessage job_messages = 1; + + // The token to obtain the next page of results if there are more. + string next_page_token = 2; + + // Autoscaling events in ascending timestamp order. + repeated AutoscalingEvent autoscaling_events = 3; +} diff --git a/third_party/googleapis/google/dataflow/v1beta3/metrics.proto b/third_party/googleapis/google/dataflow/v1beta3/metrics.proto new file mode 100644 index 0000000..823eeb7 --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/metrics.proto @@ -0,0 +1,369 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.dataflow.v1beta3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3"; +option go_package = "google.golang.org/genproto/googleapis/dataflow/v1beta3;dataflow"; +option java_multiple_files = true; +option java_outer_classname = "MetricsProto"; +option java_package = "com.google.dataflow.v1beta3"; +option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3"; +option ruby_package = "Google::Cloud::Dataflow::V1beta3"; + +// The Dataflow Metrics API lets you monitor the progress of Dataflow +// jobs. +service MetricsV1Beta3 { + option (google.api.default_host) = "dataflow.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/compute," + "https://www.googleapis.com/auth/compute.readonly," + "https://www.googleapis.com/auth/userinfo.email"; + + // Request the job status. + // + // To request the status of a job, we recommend using + // `projects.locations.jobs.getMetrics` with a [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints). Using + // `projects.jobs.getMetrics` is not recommended, as you can only request the + // status of jobs that are running in `us-central1`. + rpc GetJobMetrics(GetJobMetricsRequest) returns (JobMetrics) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}/metrics" + additional_bindings { + get: "/v1b3/projects/{project_id}/jobs/{job_id}/metrics" + } + }; + } + + // Request detailed information about the execution status of the job. + // + // EXPERIMENTAL. This API is subject to change or removal without notice. + rpc GetJobExecutionDetails(GetJobExecutionDetailsRequest) returns (JobExecutionDetails) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}/executionDetails" + }; + } + + // Request detailed information about the execution status of a stage of the + // job. + // + // EXPERIMENTAL. This API is subject to change or removal without notice. + rpc GetStageExecutionDetails(GetStageExecutionDetailsRequest) returns (StageExecutionDetails) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}/stages/{stage_id}/executionDetails" + }; + } +} + +// Identifies a metric, by describing the source which generated the +// metric. +message MetricStructuredName { + // Origin (namespace) of metric name. May be blank for user-define metrics; + // will be "dataflow" for metrics defined by the Dataflow service or SDK. + string origin = 1; + + // Worker-defined metric name. + string name = 2; + + // Zero or more labeled fields which identify the part of the job this + // metric is associated with, such as the name of a step or collection. + // + // For example, built-in counters associated with steps will have + // context['step'] = <step-name>. Counters associated with PCollections + // in the SDK will have context['pcollection'] = <pcollection-name>. + map<string, string> context = 3; +} + +// Describes the state of a metric. +message MetricUpdate { + // Name of the metric. + MetricStructuredName name = 1; + + // Metric aggregation kind. The possible metric aggregation kinds are + // "Sum", "Max", "Min", "Mean", "Set", "And", "Or", and "Distribution". + // The specified aggregation kind is case-insensitive. + // + // If omitted, this is not an aggregated value but instead + // a single metric sample value. + string kind = 2; + + // True if this metric is reported as the total cumulative aggregate + // value accumulated since the worker started working on this WorkItem. + // By default this is false, indicating that this metric is reported + // as a delta that is not associated with any WorkItem. + bool cumulative = 3; + + // Worker-computed aggregate value for aggregation kinds "Sum", "Max", "Min", + // "And", and "Or". The possible value types are Long, Double, and Boolean. + google.protobuf.Value scalar = 4; + + // Worker-computed aggregate value for the "Mean" aggregation kind. + // This holds the sum of the aggregated values and is used in combination + // with mean_count below to obtain the actual mean aggregate value. + // The only possible value types are Long and Double. + google.protobuf.Value mean_sum = 5; + + // Worker-computed aggregate value for the "Mean" aggregation kind. + // This holds the count of the aggregated values and is used in combination + // with mean_sum above to obtain the actual mean aggregate value. + // The only possible value type is Long. + google.protobuf.Value mean_count = 6; + + // Worker-computed aggregate value for the "Set" aggregation kind. The only + // possible value type is a list of Values whose type can be Long, Double, + // or String, according to the metric's type. All Values in the list must + // be of the same type. + google.protobuf.Value set = 7; + + // A struct value describing properties of a distribution of numeric values. + google.protobuf.Value distribution = 11; + + // A struct value describing properties of a Gauge. + // Metrics of gauge type show the value of a metric across time, and is + // aggregated based on the newest value. + google.protobuf.Value gauge = 12; + + // Worker-computed aggregate value for internal use by the Dataflow + // service. + google.protobuf.Value internal = 8; + + // Timestamp associated with the metric value. Optional when workers are + // reporting work progress; it will be filled in responses from the + // metrics API. + google.protobuf.Timestamp update_time = 9; +} + +// Request to get job metrics. +message GetJobMetricsRequest { + // A project id. + string project_id = 1; + + // The job to get metrics for. + string job_id = 2; + + // Return only metric data that has changed since this time. + // Default is to return all information about all metrics for the job. + google.protobuf.Timestamp start_time = 3; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains the job specified by job_id. + string location = 4; +} + +// JobMetrics contains a collection of metrics describing the detailed progress +// of a Dataflow job. Metrics correspond to user-defined and system-defined +// metrics in the job. +// +// This resource captures only the most recent values of each metric; +// time-series data can be queried for them (under the same metric names) +// from Cloud Monitoring. +message JobMetrics { + // Timestamp as of which metric values are current. + google.protobuf.Timestamp metric_time = 1; + + // All metrics for this job. + repeated MetricUpdate metrics = 2; +} + +// Request to get job execution details. +message GetJobExecutionDetailsRequest { + // A project id. + string project_id = 1; + + // The job to get execution details for. + string job_id = 2; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains the job specified by job_id. + string location = 3; + + // If specified, determines the maximum number of stages to + // return. If unspecified, the service may choose an appropriate + // default, or may return an arbitrarily large number of results. + int32 page_size = 4; + + // If supplied, this should be the value of next_page_token returned + // by an earlier call. This will cause the next page of results to + // be returned. + string page_token = 5; +} + +// Information about the progress of some component of job execution. +message ProgressTimeseries { + // A point in the timeseries. + message Point { + // The timestamp of the point. + google.protobuf.Timestamp time = 1; + + // The value of the point. + double value = 2; + } + + // The current progress of the component, in the range [0,1]. + double current_progress = 1; + + // History of progress for the component. + // + // Points are sorted by time. + repeated Point data_points = 2; +} + +// The state of some component of job execution. +enum ExecutionState { + // The component state is unknown or unspecified. + EXECUTION_STATE_UNKNOWN = 0; + + // The component is not yet running. + EXECUTION_STATE_NOT_STARTED = 1; + + // The component is currently running. + EXECUTION_STATE_RUNNING = 2; + + // The component succeeded. + EXECUTION_STATE_SUCCEEDED = 3; + + // The component failed. + EXECUTION_STATE_FAILED = 4; + + // Execution of the component was cancelled. + EXECUTION_STATE_CANCELLED = 5; +} + +// Information about a particular execution stage of a job. +message StageSummary { + // ID of this stage + string stage_id = 1; + + // State of this stage. + ExecutionState state = 2; + + // Start time of this stage. + google.protobuf.Timestamp start_time = 3; + + // End time of this stage. + // + // If the work item is completed, this is the actual end time of the stage. + // Otherwise, it is the predicted end time. + google.protobuf.Timestamp end_time = 4; + + // Progress for this stage. + // Only applicable to Batch jobs. + ProgressTimeseries progress = 5; + + // Metrics for this stage. + repeated MetricUpdate metrics = 6; +} + +// Information about the execution of a job. +message JobExecutionDetails { + // The stages of the job execution. + repeated StageSummary stages = 1; + + // If present, this response does not contain all requested tasks. To obtain + // the next page of results, repeat the request with page_token set to this + // value. + string next_page_token = 2; +} + +// Request to get information about a particular execution stage of a job. +// Currently only tracked for Batch jobs. +message GetStageExecutionDetailsRequest { + // A project id. + string project_id = 1; + + // The job to get execution details for. + string job_id = 2; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that + // contains the job specified by job_id. + string location = 3; + + // The stage for which to fetch information. + string stage_id = 4; + + // If specified, determines the maximum number of work items to + // return. If unspecified, the service may choose an appropriate + // default, or may return an arbitrarily large number of results. + int32 page_size = 5; + + // If supplied, this should be the value of next_page_token returned + // by an earlier call. This will cause the next page of results to + // be returned. + string page_token = 6; + + // Lower time bound of work items to include, by start time. + google.protobuf.Timestamp start_time = 7; + + // Upper time bound of work items to include, by start time. + google.protobuf.Timestamp end_time = 8; +} + +// Information about an individual work item execution. +message WorkItemDetails { + // Name of this work item. + string task_id = 1; + + // Attempt ID of this work item + string attempt_id = 2; + + // Start time of this work item attempt. + google.protobuf.Timestamp start_time = 3; + + // End time of this work item attempt. + // + // If the work item is completed, this is the actual end time of the work + // item. Otherwise, it is the predicted end time. + google.protobuf.Timestamp end_time = 4; + + // State of this work item. + ExecutionState state = 5; + + // Progress of this work item. + ProgressTimeseries progress = 6; + + // Metrics for this work item. + repeated MetricUpdate metrics = 7; +} + +// Information about a worker +message WorkerDetails { + // Name of this worker + string worker_name = 1; + + // Work items processed by this worker, sorted by time. + repeated WorkItemDetails work_items = 2; +} + +// Information about the workers and work items within a stage. +message StageExecutionDetails { + // Workers that have done work on the stage. + repeated WorkerDetails workers = 1; + + // If present, this response does not contain all requested tasks. To obtain + // the next page of results, repeat the request with page_token set to this + // value. + string next_page_token = 2; +} diff --git a/third_party/googleapis/google/dataflow/v1beta3/snapshots.proto b/third_party/googleapis/google/dataflow/v1beta3/snapshots.proto new file mode 100644 index 0000000..0868db8 --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/snapshots.proto @@ -0,0 +1,188 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.dataflow.v1beta3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3"; +option go_package = "google.golang.org/genproto/googleapis/dataflow/v1beta3;dataflow"; +option java_multiple_files = true; +option java_outer_classname = "SnapshotsProto"; +option java_package = "com.google.dataflow.v1beta3"; +option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3"; +option ruby_package = "Google::Cloud::Dataflow::V1beta3"; + +// Provides methods to manage snapshots of Google Cloud Dataflow jobs. +service SnapshotsV1Beta3 { + option (google.api.default_host) = "dataflow.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/compute," + "https://www.googleapis.com/auth/compute.readonly," + "https://www.googleapis.com/auth/userinfo.email"; + + // Gets information about a snapshot. + rpc GetSnapshot(GetSnapshotRequest) returns (Snapshot) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/snapshots/{snapshot_id}" + additional_bindings { + get: "/v1b3/projects/{project_id}/snapshots/{snapshot_id}" + } + }; + } + + // Deletes a snapshot. + rpc DeleteSnapshot(DeleteSnapshotRequest) returns (DeleteSnapshotResponse) { + option (google.api.http) = { + delete: "/v1b3/projects/{project_id}/locations/{location}/snapshots/{snapshot_id}" + additional_bindings { + delete: "/v1b3/projects/{project_id}/snapshots" + } + }; + } + + // Lists snapshots. + rpc ListSnapshots(ListSnapshotsRequest) returns (ListSnapshotsResponse) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/jobs/{job_id}/snapshots" + additional_bindings { + get: "/v1b3/projects/{project_id}/locations/{location}/snapshots" + } + additional_bindings { + get: "/v1b3/projects/{project_id}/snapshots" + } + }; + } +} + +// Snapshot state. +enum SnapshotState { + // Unknown state. + UNKNOWN_SNAPSHOT_STATE = 0; + + // Snapshot intent to create has been persisted, snapshotting of state has not + // yet started. + PENDING = 1; + + // Snapshotting is being performed. + RUNNING = 2; + + // Snapshot has been created and is ready to be used. + READY = 3; + + // Snapshot failed to be created. + FAILED = 4; + + // Snapshot has been deleted. + DELETED = 5; +} + +// Represents a Pubsub snapshot. +message PubsubSnapshotMetadata { + // The name of the Pubsub topic. + string topic_name = 1; + + // The name of the Pubsub snapshot. + string snapshot_name = 2; + + // The expire time of the Pubsub snapshot. + google.protobuf.Timestamp expire_time = 3; +} + +// Represents a snapshot of a job. +message Snapshot { + // The unique ID of this snapshot. + string id = 1; + + // The project this snapshot belongs to. + string project_id = 2; + + // The job this snapshot was created from. + string source_job_id = 3; + + // The time this snapshot was created. + google.protobuf.Timestamp creation_time = 4; + + // The time after which this snapshot will be automatically deleted. + google.protobuf.Duration ttl = 5; + + // State of the snapshot. + SnapshotState state = 6; + + // Pub/Sub snapshot metadata. + repeated PubsubSnapshotMetadata pubsub_metadata = 7; + + // User specified description of the snapshot. Maybe empty. + string description = 8; + + // The disk byte size of the snapshot. Only available for snapshots in READY + // state. + int64 disk_size_bytes = 9; + + // Cloud region where this snapshot lives in, e.g., "us-central1". + string region = 10; +} + +// Request to get information about a snapshot +message GetSnapshotRequest { + // The ID of the Cloud Platform project that the snapshot belongs to. + string project_id = 1; + + // The ID of the snapshot. + string snapshot_id = 2; + + // The location that contains this snapshot. + string location = 3; +} + +// Request to delete a snapshot. +message DeleteSnapshotRequest { + // The ID of the Cloud Platform project that the snapshot belongs to. + string project_id = 1; + + // The ID of the snapshot. + string snapshot_id = 2; + + // The location that contains this snapshot. + string location = 3; +} + +// Response from deleting a snapshot. +message DeleteSnapshotResponse { + +} + +// Request to list snapshots. +message ListSnapshotsRequest { + // The project ID to list snapshots for. + string project_id = 1; + + // If specified, list snapshots created from this job. + string job_id = 3; + + // The location to list snapshots in. + string location = 2; +} + +// List of snapshots. +message ListSnapshotsResponse { + // Returned snapshots. + repeated Snapshot snapshots = 1; +} diff --git a/third_party/googleapis/google/dataflow/v1beta3/streaming.proto b/third_party/googleapis/google/dataflow/v1beta3/streaming.proto new file mode 100644 index 0000000..77577ca --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/streaming.proto @@ -0,0 +1,231 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.dataflow.v1beta3; + +option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3"; +option go_package = "google.golang.org/genproto/googleapis/dataflow/v1beta3;dataflow"; +option java_multiple_files = true; +option java_outer_classname = "StreamingProto"; +option java_package = "com.google.dataflow.v1beta3"; +option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3"; +option ruby_package = "Google::Cloud::Dataflow::V1beta3"; + +// Global topology of the streaming Dataflow job, including all +// computations and their sharded locations. +message TopologyConfig { + // The computations associated with a streaming Dataflow job. + repeated ComputationTopology computations = 1; + + // The disks assigned to a streaming Dataflow job. + repeated DataDiskAssignment data_disk_assignments = 2; + + // Maps user stage names to stable computation names. + map<string, string> user_stage_to_computation_name_map = 3; + + // The size (in bits) of keys that will be assigned to source messages. + int32 forwarding_key_bits = 4; + + // Version number for persistent state. + int32 persistent_state_version = 5; +} + +// Identifies a pubsub location to use for transferring data into or +// out of a streaming Dataflow job. +message PubsubLocation { + // A pubsub topic, in the form of + // "pubsub.googleapis.com/topics/<project-id>/<topic-name>" + string topic = 1; + + // A pubsub subscription, in the form of + // "pubsub.googleapis.com/subscriptions/<project-id>/<subscription-name>" + string subscription = 2; + + // If set, contains a pubsub label from which to extract record timestamps. + // If left empty, record timestamps will be generated upon arrival. + string timestamp_label = 3; + + // If set, contains a pubsub label from which to extract record ids. + // If left empty, record deduplication will be strictly best effort. + string id_label = 4; + + // Indicates whether the pipeline allows late-arriving data. + bool drop_late_data = 5; + + // If set, specifies the pubsub subscription that will be used for tracking + // custom time timestamps for watermark estimation. + string tracking_subscription = 6; + + // If true, then the client has requested to get pubsub attributes. + bool with_attributes = 7; +} + +// Identifies the location of a streaming computation stage, for +// stage-to-stage communication. +message StreamingStageLocation { + // Identifies the particular stream within the streaming Dataflow + // job. + string stream_id = 1; +} + +// Identifies the location of a streaming side input. +message StreamingSideInputLocation { + // Identifies the particular side input within the streaming Dataflow job. + string tag = 1; + + // Identifies the state family where this side input is stored. + string state_family = 2; +} + +// Identifies the location of a custom souce. +message CustomSourceLocation { + // Whether this source is stateful. + bool stateful = 1; +} + +// Describes a stream of data, either as input to be processed or as +// output of a streaming Dataflow job. +message StreamLocation { + // A specification of a stream's location. + oneof location { + // The stream is part of another computation within the current + // streaming Dataflow job. + StreamingStageLocation streaming_stage_location = 1; + + // The stream is a pubsub stream. + PubsubLocation pubsub_location = 2; + + // The stream is a streaming side input. + StreamingSideInputLocation side_input_location = 3; + + // The stream is a custom source. + CustomSourceLocation custom_source_location = 4; + } +} + +// State family configuration. +message StateFamilyConfig { + // The state family value. + string state_family = 1; + + // If true, this family corresponds to a read operation. + bool is_read = 2; +} + +// All configuration data for a particular Computation. +message ComputationTopology { + // The system stage name. + string system_stage_name = 1; + + // The ID of the computation. + string computation_id = 5; + + // The key ranges processed by the computation. + repeated KeyRangeLocation key_ranges = 2; + + // The inputs to the computation. + repeated StreamLocation inputs = 3; + + // The outputs from the computation. + repeated StreamLocation outputs = 4; + + // The state family values. + repeated StateFamilyConfig state_families = 7; +} + +// Location information for a specific key-range of a sharded computation. +// Currently we only support UTF-8 character splits to simplify encoding into +// JSON. +message KeyRangeLocation { + // The start (inclusive) of the key range. + string start = 1; + + // The end (exclusive) of the key range. + string end = 2; + + // The physical location of this range assignment to be used for + // streaming computation cross-worker message delivery. + string delivery_endpoint = 3; + + // The name of the data disk where data for this range is stored. + // This name is local to the Google Cloud Platform project and uniquely + // identifies the disk within that project, for example + // "myproject-1014-104817-4c2-harness-0-disk-1". + string data_disk = 5; + + // DEPRECATED. The location of the persistent state for this range, as a + // persistent directory in the worker local filesystem. + string deprecated_persistent_directory = 4 [deprecated = true]; +} + +// Describes mounted data disk. +message MountedDataDisk { + // The name of the data disk. + // This name is local to the Google Cloud Platform project and uniquely + // identifies the disk within that project, for example + // "myproject-1014-104817-4c2-harness-0-disk-1". + string data_disk = 1; +} + +// Data disk assignment for a given VM instance. +message DataDiskAssignment { + // VM instance name the data disks mounted to, for example + // "myproject-1014-104817-4c2-harness-0". + string vm_instance = 1; + + // Mounted data disks. The order is important a data disk's 0-based index in + // this list defines which persistent directory the disk is mounted to, for + // example the list of { "myproject-1014-104817-4c2-harness-0-disk-0" }, + // { "myproject-1014-104817-4c2-harness-0-disk-1" }. + repeated string data_disks = 2; +} + +// Data disk assignment information for a specific key-range of a sharded +// computation. +// Currently we only support UTF-8 character splits to simplify encoding into +// JSON. +message KeyRangeDataDiskAssignment { + // The start (inclusive) of the key range. + string start = 1; + + // The end (exclusive) of the key range. + string end = 2; + + // The name of the data disk where data for this range is stored. + // This name is local to the Google Cloud Platform project and uniquely + // identifies the disk within that project, for example + // "myproject-1014-104817-4c2-harness-0-disk-1". + string data_disk = 3; +} + +// Describes full or partial data disk assignment information of the computation +// ranges. +message StreamingComputationRanges { + // The ID of the computation. + string computation_id = 1; + + // Data disk assignments for ranges from this computation. + repeated KeyRangeDataDiskAssignment range_assignments = 2; +} + +// Streaming appliance snapshot configuration. +message StreamingApplianceSnapshotConfig { + // If set, indicates the snapshot id for the snapshot being performed. + string snapshot_id = 1; + + // Indicates which endpoint is used to import appliance state. + string import_state_endpoint = 2; +} diff --git a/third_party/googleapis/google/dataflow/v1beta3/templates.proto b/third_party/googleapis/google/dataflow/v1beta3/templates.proto new file mode 100644 index 0000000..064e3a8 --- /dev/null +++ b/third_party/googleapis/google/dataflow/v1beta3/templates.proto @@ -0,0 +1,632 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.dataflow.v1beta3; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/dataflow/v1beta3/environment.proto"; +import "google/dataflow/v1beta3/jobs.proto"; +import "google/rpc/status.proto"; + +option csharp_namespace = "Google.Cloud.Dataflow.V1Beta3"; +option go_package = "google.golang.org/genproto/googleapis/dataflow/v1beta3;dataflow"; +option java_multiple_files = true; +option java_outer_classname = "TemplatesProto"; +option java_package = "com.google.dataflow.v1beta3"; +option php_namespace = "Google\\Cloud\\Dataflow\\V1beta3"; +option ruby_package = "Google::Cloud::Dataflow::V1beta3"; + +// Provides a method to create Cloud Dataflow jobs from templates. +service TemplatesService { + option (google.api.default_host) = "dataflow.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/compute," + "https://www.googleapis.com/auth/compute.readonly," + "https://www.googleapis.com/auth/userinfo.email"; + + // Creates a Cloud Dataflow job from a template. + rpc CreateJobFromTemplate(CreateJobFromTemplateRequest) returns (Job) { + option (google.api.http) = { + post: "/v1b3/projects/{project_id}/locations/{location}/templates" + body: "*" + additional_bindings { + post: "/v1b3/projects/{project_id}/templates" + body: "*" + } + }; + } + + // Launch a template. + rpc LaunchTemplate(LaunchTemplateRequest) returns (LaunchTemplateResponse) { + option (google.api.http) = { + post: "/v1b3/projects/{project_id}/locations/{location}/templates:launch" + body: "launch_parameters" + additional_bindings { + post: "/v1b3/projects/{project_id}/templates:launch" + body: "launch_parameters" + } + }; + } + + // Get the template associated with a template. + rpc GetTemplate(GetTemplateRequest) returns (GetTemplateResponse) { + option (google.api.http) = { + get: "/v1b3/projects/{project_id}/locations/{location}/templates:get" + additional_bindings { + get: "/v1b3/projects/{project_id}/templates:get" + } + }; + } +} + +// Provides a service for Flex templates. This feature is not ready yet. +service FlexTemplatesService { + option (google.api.default_host) = "dataflow.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform," + "https://www.googleapis.com/auth/compute," + "https://www.googleapis.com/auth/compute.readonly," + "https://www.googleapis.com/auth/userinfo.email"; + + // Launch a job with a FlexTemplate. + rpc LaunchFlexTemplate(LaunchFlexTemplateRequest) returns (LaunchFlexTemplateResponse) { + option (google.api.http) = { + post: "/v1b3/projects/{project_id}/locations/{location}/flexTemplates:launch" + body: "*" + }; + } +} + +// Response to the request to launch a job from Flex Template. +message LaunchFlexTemplateResponse { + // The job that was launched, if the request was not a dry run and + // the job was successfully launched. + Job job = 1; +} + +// Container Spec. +message ContainerSpec { + // Name of the docker container image. E.g., gcr.io/project/some-image + string image = 1; + + // Metadata describing a template including description and validation rules. + TemplateMetadata metadata = 2; + + // Required. SDK info of the Flex Template. + SDKInfo sdk_info = 3; + + // Default runtime environment for the job. + FlexTemplateRuntimeEnvironment default_environment = 4; +} + +// Launch FlexTemplate Parameter. +message LaunchFlexTemplateParameter { + // Required. The job name to use for the created job. For update job request, + // job name should be same as the existing running job. + string job_name = 1; + + // Launch Mechanism. + oneof template { + // Spec about the container image to launch. + ContainerSpec container_spec = 4; + + // Cloud Storage path to a file with json serialized ContainerSpec as + // content. + string container_spec_gcs_path = 5; + } + + // The parameters for FlexTemplate. + // Ex. {"num_workers":"5"} + map<string, string> parameters = 2; + + // Launch options for this flex template job. This is a common set of options + // across languages and templates. This should not be used to pass job + // parameters. + map<string, string> launch_options = 6; + + // The runtime environment for the FlexTemplate job + FlexTemplateRuntimeEnvironment environment = 7; + + // Set this to true if you are sending a request to update a running + // streaming job. When set, the job name should be the same as the + // running job. + bool update = 8; + + // Use this to pass transform_name_mappings for streaming update jobs. + // Ex:{"oldTransformName":"newTransformName",...}' + map<string, string> transform_name_mappings = 9; +} + +// The environment values to be set at runtime for flex template. +message FlexTemplateRuntimeEnvironment { + // The initial number of Google Compute Engine instances for the job. + int32 num_workers = 1; + + // The maximum number of Google Compute Engine instances to be made + // available to your pipeline during execution, from 1 to 1000. + int32 max_workers = 2; + + // The Compute Engine [availability + // zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones) + // for launching worker instances to run your pipeline. + // In the future, worker_zone will take precedence. + string zone = 3; + + // The email address of the service account to run the job as. + string service_account_email = 4; + + // The Cloud Storage path to use for temporary files. + // Must be a valid Cloud Storage URL, beginning with `gs://`. + string temp_location = 5; + + // The machine type to use for the job. Defaults to the value from the + // template if not specified. + string machine_type = 6; + + // Additional experiment flags for the job. + repeated string additional_experiments = 7; + + // Network to which VMs will be assigned. If empty or unspecified, + // the service will use the network "default". + string network = 8; + + // Subnetwork to which VMs will be assigned, if desired. You can specify a + // subnetwork using either a complete URL or an abbreviated path. Expected to + // be of the form + // "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK" + // or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in + // a Shared VPC network, you must use the complete URL. + string subnetwork = 9; + + // Additional user labels to be specified for the job. + // Keys and values must follow the restrictions specified in the [labeling + // restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions) + // page. + // An object containing a list of "key": value pairs. + // Example: { "name": "wrench", "mass": "1kg", "count": "3" }. + map<string, string> additional_user_labels = 10; + + // Name for the Cloud KMS key for the job. + // Key format is: + // projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key> + string kms_key_name = 11; + + // Configuration for VM IPs. + WorkerIPAddressConfiguration ip_configuration = 12; + + // The Compute Engine region + // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in + // which worker processing should occur, e.g. "us-west1". Mutually exclusive + // with worker_zone. If neither worker_region nor worker_zone is specified, + // default to the control plane's region. + string worker_region = 13; + + // The Compute Engine zone + // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in + // which worker processing should occur, e.g. "us-west1-a". Mutually exclusive + // with worker_region. If neither worker_region nor worker_zone is specified, + // a zone in the control plane's region is chosen based on available capacity. + // If both `worker_zone` and `zone` are set, `worker_zone` takes precedence. + string worker_zone = 14; + + // Whether to enable Streaming Engine for the job. + bool enable_streaming_engine = 15; + + // Set FlexRS goal for the job. + // https://cloud.google.com/dataflow/docs/guides/flexrs + FlexResourceSchedulingGoal flexrs_goal = 16; + + // The Cloud Storage path for staging local files. + // Must be a valid Cloud Storage URL, beginning with `gs://`. + string staging_location = 17; + + // Docker registry location of container image to use for the 'worker harness. + // Default is the container for the version of the SDK. Note this field is + // only valid for portable pipelines. + string sdk_container_image = 18; + + // Worker disk size, in gigabytes. + int32 disk_size_gb = 20; + + // The algorithm to use for autoscaling + AutoscalingAlgorithm autoscaling_algorithm = 21; + + // If true, save a heap dump before killing a thread or process which is GC + // thrashing or out of memory. The location of the heap file will either be + // echoed back to the user, or the user will be given the opportunity to + // download the heap file. + bool dump_heap_on_oom = 22; + + // Cloud Storage bucket (directory) to upload heap dumps to the given + // location. Enabling this implies that heap dumps should be generated on OOM + // (dump_heap_on_oom is set to true). + string save_heap_dumps_to_gcs_path = 23; + + // The machine type to use for launching the job. The default is + // n1-standard-1. + string launcher_machine_type = 24; +} + +// A request to launch a Cloud Dataflow job from a FlexTemplate. +message LaunchFlexTemplateRequest { + // Required. The ID of the Cloud Platform project that the job belongs to. + string project_id = 1; + + // Required. Parameter to launch a job form Flex Template. + LaunchFlexTemplateParameter launch_parameter = 2; + + // Required. The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to + // which to direct the request. E.g., us-central1, us-west1. + string location = 3; + + // If true, the request is validated but not actually executed. + // Defaults to false. + bool validate_only = 4; +} + +// The environment values to set at runtime. +message RuntimeEnvironment { + // The initial number of Google Compute Engine instnaces for the job. + int32 num_workers = 11; + + // The maximum number of Google Compute Engine instances to be made + // available to your pipeline during execution, from 1 to 1000. + int32 max_workers = 1; + + // The Compute Engine [availability + // zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones) + // for launching worker instances to run your pipeline. + // In the future, worker_zone will take precedence. + string zone = 2; + + // The email address of the service account to run the job as. + string service_account_email = 3; + + // The Cloud Storage path to use for temporary files. + // Must be a valid Cloud Storage URL, beginning with `gs://`. + string temp_location = 4; + + // Whether to bypass the safety checks for the job's temporary directory. + // Use with caution. + bool bypass_temp_dir_validation = 5; + + // The machine type to use for the job. Defaults to the value from the + // template if not specified. + string machine_type = 6; + + // Additional experiment flags for the job, specified with the + // `--experiments` option. + repeated string additional_experiments = 7; + + // Network to which VMs will be assigned. If empty or unspecified, + // the service will use the network "default". + string network = 8; + + // Subnetwork to which VMs will be assigned, if desired. You can specify a + // subnetwork using either a complete URL or an abbreviated path. Expected to + // be of the form + // "https://www.googleapis.com/compute/v1/projects/HOST_PROJECT_ID/regions/REGION/subnetworks/SUBNETWORK" + // or "regions/REGION/subnetworks/SUBNETWORK". If the subnetwork is located in + // a Shared VPC network, you must use the complete URL. + string subnetwork = 9; + + // Additional user labels to be specified for the job. + // Keys and values should follow the restrictions specified in the [labeling + // restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions) + // page. + // An object containing a list of "key": value pairs. + // Example: { "name": "wrench", "mass": "1kg", "count": "3" }. + map<string, string> additional_user_labels = 10; + + // Name for the Cloud KMS key for the job. + // Key format is: + // projects/<project>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key> + string kms_key_name = 12; + + // Configuration for VM IPs. + WorkerIPAddressConfiguration ip_configuration = 14; + + // The Compute Engine region + // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in + // which worker processing should occur, e.g. "us-west1". Mutually exclusive + // with worker_zone. If neither worker_region nor worker_zone is specified, + // default to the control plane's region. + string worker_region = 15; + + // The Compute Engine zone + // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in + // which worker processing should occur, e.g. "us-west1-a". Mutually exclusive + // with worker_region. If neither worker_region nor worker_zone is specified, + // a zone in the control plane's region is chosen based on available capacity. + // If both `worker_zone` and `zone` are set, `worker_zone` takes precedence. + string worker_zone = 16; + + // Whether to enable Streaming Engine for the job. + bool enable_streaming_engine = 17; +} + +// Metadata for a specific parameter. +message ParameterMetadata { + // Required. The name of the parameter. + string name = 1; + + // Required. The label to display for the parameter. + string label = 2; + + // Required. The help text to display for the parameter. + string help_text = 3; + + // Optional. Whether the parameter is optional. Defaults to false. + bool is_optional = 4; + + // Optional. Regexes that the parameter must match. + repeated string regexes = 5; + + // Optional. The type of the parameter. + // Used for selecting input picker. + ParameterType param_type = 6; + + // Optional. Additional metadata for describing this parameter. + map<string, string> custom_metadata = 7; +} + +// ParameterType specifies what kind of input we need for this parameter. +enum ParameterType { + // Default input type. + DEFAULT = 0; + + // The parameter specifies generic text input. + TEXT = 1; + + // The parameter specifies a Cloud Storage Bucket to read from. + GCS_READ_BUCKET = 2; + + // The parameter specifies a Cloud Storage Bucket to write to. + GCS_WRITE_BUCKET = 3; + + // The parameter specifies a Cloud Storage file path to read from. + GCS_READ_FILE = 4; + + // The parameter specifies a Cloud Storage file path to write to. + GCS_WRITE_FILE = 5; + + // The parameter specifies a Cloud Storage folder path to read from. + GCS_READ_FOLDER = 6; + + // The parameter specifies a Cloud Storage folder to write to. + GCS_WRITE_FOLDER = 7; + + // The parameter specifies a Pub/Sub Topic. + PUBSUB_TOPIC = 8; + + // The parameter specifies a Pub/Sub Subscription. + PUBSUB_SUBSCRIPTION = 9; +} + +// Metadata describing a template. +message TemplateMetadata { + // Required. The name of the template. + string name = 1; + + // Optional. A description of the template. + string description = 2; + + // The parameters for the template. + repeated ParameterMetadata parameters = 3; +} + +// SDK Information. +message SDKInfo { + // SDK Language. + enum Language { + // UNKNOWN Language. + UNKNOWN = 0; + + // Java. + JAVA = 1; + + // Python. + PYTHON = 2; + } + + // Required. The SDK Language. + Language language = 1; + + // Optional. The SDK version. + string version = 2; +} + +// RuntimeMetadata describing a runtime environment. +message RuntimeMetadata { + // SDK Info for the template. + SDKInfo sdk_info = 1; + + // The parameters for the template. + repeated ParameterMetadata parameters = 2; +} + +// A request to create a Cloud Dataflow job from a template. +message CreateJobFromTemplateRequest { + // Required. The ID of the Cloud Platform project that the job belongs to. + string project_id = 1; + + // Required. The job name to use for the created job. + string job_name = 4; + + // The template from which to create the job. + oneof template { + // Required. A Cloud Storage path to the template from which to + // create the job. + // Must be a valid Cloud Storage URL, beginning with `gs://`. + string gcs_path = 2; + } + + // The runtime parameters to pass to the job. + map<string, string> parameters = 3; + + // The runtime environment for the job. + RuntimeEnvironment environment = 5; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to + // which to direct the request. + string location = 6; +} + +// A request to retrieve a Cloud Dataflow job template. +message GetTemplateRequest { + // The various views of a template that may be retrieved. + enum TemplateView { + // Template view that retrieves only the metadata associated with the + // template. + METADATA_ONLY = 0; + } + + // Required. The ID of the Cloud Platform project that the job belongs to. + string project_id = 1; + + // The template from which to create the job. + oneof template { + // Required. A Cloud Storage path to the template from which to + // create the job. + // Must be valid Cloud Storage URL, beginning with 'gs://'. + string gcs_path = 2; + } + + // The view to retrieve. Defaults to METADATA_ONLY. + TemplateView view = 3; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to + // which to direct the request. + string location = 4; +} + +// The response to a GetTemplate request. +message GetTemplateResponse { + // Template Type. + enum TemplateType { + // Unknown Template Type. + UNKNOWN = 0; + + // Legacy Template. + LEGACY = 1; + + // Flex Template. + FLEX = 2; + } + + // The status of the get template request. Any problems with the + // request will be indicated in the error_details. + google.rpc.Status status = 1; + + // The template metadata describing the template name, available + // parameters, etc. + TemplateMetadata metadata = 2; + + // Template Type. + TemplateType template_type = 3; + + // Describes the runtime metadata with SDKInfo and available parameters. + RuntimeMetadata runtime_metadata = 4; +} + +// Parameters to provide to the template being launched. +message LaunchTemplateParameters { + // Required. The job name to use for the created job. + string job_name = 1; + + // The runtime parameters to pass to the job. + map<string, string> parameters = 2; + + // The runtime environment for the job. + RuntimeEnvironment environment = 3; + + // If set, replace the existing pipeline with the name specified by jobName + // with this pipeline, preserving state. + bool update = 4; + + // Only applicable when updating a pipeline. Map of transform name prefixes of + // the job to be replaced to the corresponding name prefixes of the new job. + map<string, string> transform_name_mapping = 5; +} + +// A request to launch a template. +message LaunchTemplateRequest { + // Required. The ID of the Cloud Platform project that the job belongs to. + string project_id = 1; + + // If true, the request is validated but not actually executed. + // Defaults to false. + bool validate_only = 2; + + // The template from which to create the job. + oneof template { + // A Cloud Storage path to the template from which to create + // the job. + // Must be valid Cloud Storage URL, beginning with 'gs://'. + string gcs_path = 3; + + // Params for launching a dynamic template. + DynamicTemplateLaunchParams dynamic_template = 6; + } + + // The parameters of the template to launch. This should be part of the + // body of the POST request. + LaunchTemplateParameters launch_parameters = 4; + + // The [regional endpoint] + // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to + // which to direct the request. + string location = 5; +} + +// Response to the request to launch a template. +message LaunchTemplateResponse { + // The job that was launched, if the request was not a dry run and + // the job was successfully launched. + Job job = 1; +} + +// Used in the error_details field of a google.rpc.Status message, this +// indicates problems with the template parameter. +message InvalidTemplateParameters { + // A specific template-parameter violation. + message ParameterViolation { + // The parameter that failed to validate. + string parameter = 1; + + // A description of why the parameter failed to validate. + string description = 2; + } + + // Describes all parameter violations in a template request. + repeated ParameterViolation parameter_violations = 1; +} + +// Params which should be passed when launching a dynamic template. +message DynamicTemplateLaunchParams { + // Path to dynamic template spec file on Cloud Storage. + // The file must be a Json serialized DynamicTemplateFieSpec object. + string gcs_path = 1; + + // Cloud Storage path for staging dependencies. + // Must be a valid Cloud Storage URL, beginning with `gs://`. + string staging_location = 2; +} |