buildbucket/proto/project_config.proto

// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Schemas for project configs.

syntax = "proto3";

package buildbucket;

option go_package = "go.chromium.org/luci/buildbucket/proto;buildbucketpb";

import "google/protobuf/duration.proto";
import "google/protobuf/wrappers.proto";

import "go.chromium.org/luci/buildbucket/proto/common.proto";
import "go.chromium.org/luci/common/proto/options.proto";
import "go.chromium.org/luci/resultdb/proto/v1/invocation.proto";

option (luci.file_metadata) = {
  doc_url: "https://config.luci.app/schemas/projects:buildbucket.cfg";
};

// Deprecated in favor of LUCI Realms. This proto is totally unused now, exists
// only to not break older configs that still may have deprecated fields
// populated.
message Acl {
  enum Role {
    READER = 0;
    SCHEDULER = 1;
    WRITER = 2;
  }
  Role role = 1 [deprecated = true];
  string group = 2 [deprecated = true];
  string identity = 3 [deprecated = true];
}

// Defines a swarmbucket builder. A builder has a name, a category and specifies
// what should happen if a build is scheduled to that builder.
//
// SECURITY WARNING: if adding more fields to this message, keep in mind that
// a user that has permissions to schedule a build to the bucket, can override
// this config.
//
// Next tag: 40.
message BuilderConfig {
  reserved 8;  // cipd_packages
  reserved 11; // build_numbers of the old format.
  reserved 13; // auto_builder_dimension of the old format.
  reserved 15; // experimental of the old format.
  reserved 19; // luci_migration_host
  reserved 27; // description of the old format.

  // Describes a cache directory persisted on a bot.
  // Prerequisite reading in BuildInfra.Swarming.CacheEntry message in
  // build.proto.
  //
  // To share a builder cache among multiple builders, it can be overridden:
  //
  //   builders {
  //     name: "a"
  //     caches {
  //       path: "builder"
  //       name: "my_shared_cache"
  //     }
  //   }
  //   builders {
  //     name: "b"
  //     caches {
  //       path: "builder"
  //       name: "my_shared_cache"
  //     }
  //   }
  //
  // Builders "a" and "b" share their builder cache. If an "a" build ran on a
  // bot and left some files in the builder cache and then a "b" build runs on
  // the same bot, the same files will be available in the builder cache.
  message CacheEntry {
    // Identifier of the cache. Length is limited to 128.
    // Defaults to path.
    // See also BuildInfra.Swarming.CacheEntry.name in build.proto.
    string name = 1;

    // Relative path where the cache in mapped into. Required.
    // See also BuildInfra.Swarming.CacheEntry.path in build.proto.
    string path = 2;

    // Number of seconds to wait for a bot with a warm cache to pick up the
    // task, before falling back to a bot with a cold (non-existent) cache.
    // See also BuildInfra.Swarming.CacheEntry.wait_for_warm_cache in build.proto.
    // The value must be multiples of 60 seconds.
    int32 wait_for_warm_cache_secs = 3;

    // Environment variable with this name will be set to the path to the cache
    // directory.
    string env_var = 4;
  }

  // DEPRECATED. See BuilderConfig.executable and BuilderConfig.properties
  //
  // To specify a recipe name, pass "$recipe_engine" property which is a JSON
  // object having "recipe" property.
  message Recipe {
    // Deprecated "repository" mode used git to bootstrap recipes directly from
    // the recipe repo. Use `cipd_package` ++ `cipd_version` below instead, as
    // they are faster, more reliable, and more flexible (because they could be
    // used to provide non-git, non-python recipes).
    reserved 1; // repository

    // Name of the recipe to run.
    string name = 2;

    // The CIPD package to fetch the recipes from.
    //
    // Typically the package will look like:
    //
    //   infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build
    //
    // Recipes bundled from internal repositories are typically under
    // `infra_internal/recipe_bundles/...`.
    //
    // But if you're building your own recipe bundles, they could be located
    // elsewhere.
    string cipd_package = 6;

    // The CIPD version to fetch. This can be a lower-cased git ref (like
    // `refs/heads/main` or `head`), or it can be a cipd tag (like
    // `git_revision:dead...beef`).
    //
    // The default is `head`, which corresponds to the git repo's HEAD ref. This
    // is typically (but not always) a symbolic ref for `refs/heads/master`.
    string cipd_version = 5;

    // Colon-separated build properties to set.
    // Ignored if BuilderConfig.properties is set.
    //
    // Use this field for string properties and use properties_j for other
    // types.
    repeated string properties = 3;

    // Same as properties, but the value must valid JSON. For example
    //   properties_j: "a:1"
    // means property a is a number 1, not string "1".
    //
    // If null, it means no property must be defined. In particular, it removes
    // a default value for the property, if any.
    //
    // Fields properties and properties_j can be used together, but cannot both
    // specify values for same property.
    repeated string properties_j = 4;
  }

  // ResultDB-specific information for a builder.
  message ResultDB {

    // Whether to enable ResultDB:Buildbucket integration.
    bool enable = 1;

    // Configuration for exporting test results to BigQuery.
    // This can have multiple values to export results to multiple BigQuery
    // tables, or to support multiple test result predicates.
    repeated luci.resultdb.v1.BigQueryExport bq_exports = 2;

    // Deprecated. Any values specified here are ignored.
    luci.resultdb.v1.HistoryOptions history_options = 3;
  }

  // Buildbucket backend-specific information for a builder.
  message Backend {
    // URI for this backend, e.g. "swarming://chromium-swarm".
    string target = 1;

    // A string interpreted as JSON encapsulating configuration for this
    // backend.
    // TODO(crbug.com/1042991): Move priority, wait_for_capacity, etc. here.
    string config_json = 2 [(luci.text_pb_format) = JSON];
  }

  // Name of the builder.
  //
  // If a builder name, will be propagated to "builder" build tag and
  // "buildername" recipe property.
  //
  // A builder name must be unique within the bucket, and match regex
  // ^[a-zA-Z0-9\-_.\(\) ]{1,128}$.
  string name = 1;

  // Backend for this builder.
  // If unset, builds are scheduled using the legacy pipeline.
  Backend backend = 32;

  // Alternate backend to use for this builder when the
  // "luci.buildbucket.backend_alt" experiment is enabled. Works even when
  // `backend` is empty. Useful for migrations to new backends.
  Backend backend_alt = 33;

  // Hostname of the swarming instance, e.g. "chromium-swarm.appspot.com".
  // Required, but defaults to deprecated Swarming.hostname.
  string swarming_host = 21;

  reserved 10; // mixins

  // Builder category. Will be used for visual grouping, for example in Code Review.
  string category = 6;

  // DEPRECATED.
  // Used only to enable "vpython:native-python-wrapper"
  // Does NOT actually propagate to swarming.
  repeated string swarming_tags = 2;

  // A requirement for a bot to execute the build.
  //
  // Supports 2 forms:
  // - "<key>:<value>" - require a bot with this dimension.
  //   This is a shortcut for "0:<key>:<value>", see below.
  // - "<expiration_secs>:<key>:<value>" - wait for up to expiration_secs.
  //   for a bot with the dimension.
  //   Supports multiple values for different keys and expiration_secs.
  //   expiration_secs must be a multiple of 60.
  //
  // If this builder is defined in a bucket, dimension "pool" is defaulted
  // to the name of the bucket. See Bucket message below.
  repeated string dimensions = 3;

  // Specifies that a recipe to run.
  // DEPRECATED: use exe.
  Recipe recipe = 4;

  // What to run when a build is ready to start.
  buildbucket.v2.Executable exe = 23;

  // A JSON object representing Build.input.properties.
  // Individual object properties can be overridden with
  // ScheduleBuildRequest.properties.
  string properties = 24 [(luci.text_pb_format) = JSON];

  // A list of top-level property names which can be overridden in
  // ScheduleBuildRequest.
  //
  // If this field is the EXACT value `["*"]` then all properties are permitted
  // to be overridden.
  //
  // NOTE: Some executables (such as the recipe engine) can have drastic
  // behavior differences based on some properties (for example, the "recipe"
  // property). If you allow the "recipe" property to be overridden, then anyone
  // with the 'buildbucket.builds.add' permission could create a Build for this
  // Builder running a different recipe (from the same recipe repo).
  repeated string allowed_property_overrides = 34;

  // Swarming task priority.
  // A value between 20 and 255, inclusive.
  // Lower means more important.
  //
  // The default value is configured in
  // https://chrome-internal.googlesource.com/infradata/config/+/master/configs/cr-buildbucket/swarming_task_template.json
  //
  // See also https://chromium.googlesource.com/infra/luci/luci-py.git/+/master/appengine/swarming/doc/User-Guide.md#request
  uint32 priority = 5;

  // Maximum build execution time.
  //
  // Not to be confused with pending time.
  //
  // If the timeout is reached, the task will be signaled according to the
  // `deadline` section of
  // https://chromium.googlesource.com/infra/luci/luci-py/+/HEAD/client/LUCI_CONTEXT.md
  // and status_details.timeout is set.
  //
  // The task will have `grace_period` amount of time to handle cleanup
  // before being forcefully terminated.
  //
  // NOTE: This corresponds with Build.execution_timeout and
  // ScheduleBuildRequest.execution_timeout; The name `execution_timeout_secs` and
  // uint32 type are relics of the past.
  uint32 execution_timeout_secs = 7;

  // Maximum amount of time to wait for the next heartbeat(i.e UpdateBuild).
  //
  // After a build is started, the client can send heartbeat requests
  // periodically. Buildbucket will mark the build as INFRA_FAILURE, if the
  // timeout threshold reaches. It’s to fail a build more quickly, rather than
  // waiting for `execution_timeout_secs` to expire. Some V1 users, which don't
  // have real task backends, can utilize this feature.
  //
  // By default, the value is 0, which means no timeout threshold is applied.
  //
  // Note: this field only takes effect for TaskBackendLite builds. For builds
  // with full-featured TaskBackend Implementation, `sync_backend_tasks` cron
  // job fulfills the similar functionality.
  uint32 heartbeat_timeout_secs = 39;

  // Maximum build pending time.
  //
  // If the timeout is reached, the build is marked as INFRA_FAILURE status
  // and both status_details.{timeout, resource_exhaustion} are set.
  //
  // NOTE: This corresponds with Build.scheduling_timeout and
  // ScheduleBuildRequest.scheduling_timeout; The name `expiration_secs` and
  // uint32 type are relics of the past.
  uint32 expiration_secs = 20;

  // Amount of cleanup time after execution_timeout_secs.
  //
  // After being signaled according to execution_timeout_secs, the task will
  // have this many seconds to clean up before being forcefully terminated.
  //
  // The signalling process is explained in the `deadline` section of
  // https://chromium.googlesource.com/infra/luci/luci-py/+/HEAD/client/LUCI_CONTEXT.md.
  //
  // Defaults to 30s if unspecified or 0.
  google.protobuf.Duration grace_period = 31;

  // If YES, will request that swarming wait until it sees at least one bot
  // report a superset of the requested dimensions.
  //
  // If UNSET/NO (the default), swarming will immediately reject a build which
  // specifies a dimension set that it's never seen before.
  //
  // Usually you want this to be UNSET/NO, unless you know that some external
  // system is working to add bots to swarming which will match the requested
  // dimensions within expiration_secs. Otherwise you'll have to wait for all of
  // `expiration_secs` until swarming tells you "Sorry, nothing has dimension
  // `os:MadeUpOS`".
  buildbucket.v2.Trinary wait_for_capacity = 29;

  // Caches that should be present on the bot.
  repeated CacheEntry caches = 9;

  // If YES, generate monotonically increasing contiguous numbers for each
  // build, unique within the builder.
  // Note: this limits the build creation rate in this builder to 5 per second.
  Toggle build_numbers = 16;

  // Email of a service account to run the build as or literal 'bot' string to
  // use Swarming bot's account (if available). Passed directly to Swarming.
  // Subject to Swarming's ACLs.
  string service_account = 12;

  // If YES, each builder will get extra dimension "builder:<builder name>"
  // added. Default is UNSET.
  //
  // For example, this config
  //
  //   builder {
  //     name: "linux-compiler"
  //     dimension: "builder:linux-compiler"
  //   }
  //
  // is equivalent to this:
  //
  //   builders {
  //     name: "linux-compiler"
  //     auto_builder_dimension: YES
  //   }
  //
  // (see also http://docs.buildbot.net/0.8.9/manual/cfg-properties.html#interpolate)
  // but are currently against complicating config with this.
  Toggle auto_builder_dimension = 17;

  // DEPRECATED
  //
  // Set the "luci.non_production" experiment in the 'experiments' field below,
  // instead.
  //
  // If YES, sets the "luci.non_production" experiment to 100% for
  // builds on this builder.
  //
  // See the documentation on `experiments` for more details about the
  // "luci.non_production" experiment.
  Toggle experimental = 18;

  // DEPRECATED
  //
  // Set the "luci.buildbucket.canary_software" experiment in the 'experiments'
  // field below, instead.
  //
  // Percentage of builds that should use a canary swarming task template.
  // A value from 0 to 100.
  // If omitted, a global server-defined default percentage is used.
  google.protobuf.UInt32Value task_template_canary_percentage = 22;

  // A mapping of experiment name to the percentage chance (0-100) that it will
  // apply to builds generated from this builder. Experiments are simply strings
  // which various parts of the stack (from LUCI services down to your build
  // scripts) may react to in order to enable certain functionality.
  //
  // You may set any experiments you like, but experiments beginning with
  // "luci." are reserved. Experiment names must conform to
  //
  //    [a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*
  //
  // Any experiments which are selected for a build show up in
  // `Build.input.experiments`.
  //
  // Its recommended that you confine your experiments to smaller, more explicit
  // targets. For example, prefer the experiment named
  // "my_project.use_mysoftware_v2_crbug_999999" rather than "use_next_gen".
  //
  // It is NOT recommended to 'piggy-back' on top of existing experiment names
  // for a different purpose. However if you want to, you can have your build
  // treat the presence of ANY experiment as equivalent to "luci.non_production"
  // being set for your build (i.e. "if any experiment is set, don't affect
  // production"). This is ulimately up to you, however.
  //
  // Well-known experiments
  //
  // Buildbucket has a number of 'global' experiments which are in various
  // states of deployment at any given time. For the current state, see
  // go/buildbucket-settings.cfg.
  map<string, int32> experiments = 28;

  // This field will set the default value of the "critical" field of
  // all the builds of this builder. Please refer to build.proto for
  // the meaning of this field.
  //
  // This value can be overridden by ScheduleBuildRequest.critical
  buildbucket.v2.Trinary critical = 25;

  // Used to enable and configure ResultDB integration.
  ResultDB resultdb = 26;

  // Description that helps users understand the purpose of the builder, in
  // HTML.
  string description_html = 30;

  // Configurations that need to be replaced when running a led build for this
  // Builder.
  //
  // Note: Builders in a dynamic bucket cannot have ShadowBuilderAdjustments.
  message ShadowBuilderAdjustments {
    string service_account = 1;
    string pool = 2;
    // A JSON object contains properties to override Build.input.properties
    // when creating the led build.
    // Same as ScheduleBuild, the top-level properties here will override the
    // ones in builder config, instead of deep merge.
    string properties = 3 [(luci.text_pb_format) = JSON];

    // Overrides default dimensions defined by builder config.
    // Same as ScheduleBuild,
    // * dimensions with empty value will be excluded.
    // * same key dimensions with both empty and non-empty values are disallowed.
    //
    // Note: for historical reason, pool can be adjusted individually.
    // If pool is adjusted individually, the same change should be reflected in
    // dimensions, and vice versa.
    repeated string dimensions = 4;
  }

  ShadowBuilderAdjustments shadow_builder_adjustments = 35;

  // This field will set the default value of the "retriable" field of
  // all the builds of this builder. Please refer to build.proto for
  // the meaning of this field.
  //
  // This value can be overridden by ScheduleBuildRequest.retriable
  buildbucket.v2.Trinary retriable = 36;

  message BuilderHealthLinks {
    // Mapping of username domain to clickable link for documentation on the health
    // metrics and how they were calculated.
    //
    // The empty domain value will be used as a fallback for anonymous users, or
    // if the user identity domain doesn't have a matching entry in this map.
    //
    // If linking an internal google link (say g3doc), use a go-link instead of a
    // raw url.
    map<string, string> doc_links = 1;
    // Mapping of username domain to clickable link for data visualization or
    // dashboards for the health metrics.
    //
    // Similar to doc_links, the empty domain value will be used as a fallback for
    // anonymous users, or if the user identity domain doesn't have a matching
    // entry in this map.
    //
    // If linking an internal google link (say g3doc), use a go-link instead of a
    // raw url.
    map<string, string> data_links = 2;
  }

  BuilderHealthLinks builder_health_metrics_links = 37;

  // The owning team's contact email. This team is responsible for fixing
  // any builder health issues (see Builder.Metadata.HealthSpec).
  // Will be validated as an email address, but nothing else.
  // It will display on milo and could be public facing, so please don't put anything sensitive.
  string contact_team_email = 38;
}

// Configuration of buildbucket-swarming integration for one bucket.
message Swarming {
  reserved 1; // hostname
  reserved 2; // url_format
  reserved 3; // builder_defaults

  // Configuration for each builder.
  // Swarming tasks are created only for builds for builders that are not
  // explicitly specified.
  repeated BuilderConfig builders = 4;

  // DEPRECATED. Use builder_defaults.task_template_canary_percentage instead.
  // Setting this field sets builder_defaults.task_template_canary_percentage.
  google.protobuf.UInt32Value task_template_canary_percentage = 5;
}

// Defines one bucket in buildbucket.cfg
message Bucket {
  reserved 4; // acl_sets

  // Name of the bucket. Names are unique within one instance of buildbucket.
  // If another project already uses this name, a config will be rejected.
  // Name reservation is first-come first-serve.
  // Regex: ^[a-z0-9\-_.]{1,100}$
  string name = 1;
  // Deprecated and ignored. Use Realms ACLs instead.
  repeated Acl acls = 2 [deprecated = true];
  // Buildbucket-swarming integration.
  // Mutually exclusive with builder_template.
  Swarming swarming = 3;

  // Name of this bucket's shadow bucket for the led builds to use.
  //
  // If omitted, it implies that led builds of this bucket reuse this bucket.
  // This is allowed, but note that it means the led builds will be in
  // the same bucket/builder with the real builds, which means Any users with
  // led access will be able to do ANYTHING that this bucket's bots and
  // service_accounts can do.
  //
  // It could also be noisy, such as:
  // * On the LUCI UI, led builds will show under the same builder as the real builds,
  // * Led builds will share the same ResultDB config as the real builds, so
  //   their test results will be exported to the same BigQuery tables.
  // * Subscribers of Buildbucket PubSub need to filter them out.
  //
  // Note: Don't set it if it's a dynamic bucket. Currently, a dynamic bucket is
  // not allowed to have a shadow bucket.
  string shadow = 5;

  // Constraints for a bucket.
  //
  // Buildbucket.CreateBuild will validate the incoming requests to make sure
  // they meet these constraints.
  message Constraints {
    // Constraints allowed pools.
    // Builds in this bucket must have a "pool" dimension which matches an entry in this list.
    repeated string pools = 1;

    // Only service accounts in this list are allowed.
    repeated string service_accounts = 2;
  }

  // Security constraints of the bucket.
  //
  // This field is used by CreateBuild on this bucket to constrain proposed
  // Builds. If a build doesn't meet the constraints, it will be rejected.
  // For shadow buckets, this is what prevents the bucket from allowing
  // totally arbitrary Builds.
  //
  // `lucicfg` will automatically populate this for the "primary" bucket
  // when using `luci.builder`.
  //
  // Buildbuceket.CreateBuild will validate the incoming requests to make sure
  // they meet these constraints.
  Constraints constraints = 6;

  // Template of builders in a dynamic bucket.
  message DynamicBuilderTemplate {
    // The Builder template which is shared among all builders in this dynamic
    // bucket.
    BuilderConfig template = 1;
  }

  // Template of builders in a dynamic bucket.
  // Mutually exclusive with swarming.
  //
  // If is not nil, the bucket is a dynamic LUCI bucket.
  // If a bucket has both swarming and dynamic_builder_template as nil,
  // the bucket is a legacy one.
  DynamicBuilderTemplate dynamic_builder_template = 7;
}

// Schema of buildbucket.cfg file, a project config.
message BuildbucketCfg {
  reserved 2; // acl_sets
  reserved 3; // builder_mixins
  reserved 4; // might be taken by builds_notification_topics

  // All buckets defined for this project.
  repeated Bucket buckets = 1;

  message Topic {
    // Topic name format should be like
    // "projects/<projid>/topics/<topicid>" and conforms to the guideline:
    // https://cloud.google.com/pubsub/docs/admin#resource_names.
    string name = 1;

    // The compression method that  `build_large_fields` uses in pubsub message
    // data. By default, it's ZLIB as this is the most common one and is the
    // built-in lib in many programming languages.
    buildbucket.v2.Compression compression = 2;
  }

  message CommonConfig {
    // A list of PubSub topics that Buildbucket will publish notifications for
    // build status changes in this project.
    // The message data schema can be found in message `BuildsV2PubSub` in
    // https://chromium.googlesource.com/infra/luci/luci-go/+/main/buildbucket/proto/notification.proto
    // Attributes on the pubsub messages:
    // - "project"
    // - "bucket"
    // - "builder"
    // - "is_completed" (The value is either "true" or "false" in string.)
    //
    // Note: `pubsub.topics.publish` permission must be granted to the
    // corresponding luci-project-scoped accounts in the cloud project(s) hosting
    // the topics.
    repeated Topic builds_notification_topics = 1;
  }

  // Global configs are shared among all buckets and builders defined inside
  // this project.
  CommonConfig common_config = 5;
}

// Toggle is a boolean with an extra state UNSET.
// When protobuf messages are merged, UNSET does not overwrite an existing
// value.
// TODO(nodir): replace with Trinary in ../common.proto.
enum Toggle {
  UNSET = 0;
  YES = 1;
  NO = 2;
}