feat(arc): let a profile run on burst nodes only

plugins/filter/arc.py

@@ -69,7 +69,7 @@
 _CONDITION_TAINT_PREFIX = "node.kubernetes.io/"
 
 # A profile's burst keys (see _burst), and the fields of a Kubernetes toleration (core/v1 Toleration).
-_BURST_KEYS = frozenset({"node_label_key", "node_label_values", "tolerations", "max_runners"})
+_BURST_KEYS = frozenset({"node_label_key", "node_label_values", "tolerations", "max_runners", "exclusive"})
 _TOLERATION_KEYS = frozenset({"key", "operator", "value", "effect", "tolerationSeconds"})
 # The highest weight a preferred node affinity term may carry (1-100 in the Kubernetes API), so keeping runner pods on the fixed nodes outweighs any other preference the scheduler scores.
 _PREFER_FIXED_NODES_WEIGHT = 100
@@ -397,12 +397,19 @@ def _burst(profile: Mapping[str, Any], where: str, errors: list[str]) -> dict[st
     if max_runners is None or max_runners < 0 or max_runners != max_runners.to_integral_value():
         errors.append(f"{where}.burst.max_runners must be a whole number of at least 0 (got {burst.get('max_runners')!r})")
         max_runners = Decimal(0)
-    return {"node_label_key": label_key, "node_label_values": list(label_values), "tolerations": [dict(toleration) for toleration in tolerations], "max_runners": int(max_runners)}
+    try:
+        exclusive = boolean(burst.get("exclusive", False), strict=True)
+    except TypeError:
+        errors.append(f"{where}.burst.exclusive must be a boolean")
+        exclusive = False
+    return {"node_label_key": label_key, "node_label_values": list(label_values), "tolerations": [dict(toleration) for toleration in tolerations], "max_runners": int(max_runners), "exclusive": exclusive}
 
 
 def arc_runner_placement(profile: Mapping[str, Any], eligibility: Mapping[str, Any]) -> dict[str, Any]:
     """Return where a profile's runner pods may be scheduled, as pod spec fields: nodeSelector, and with burst also affinity and tolerations.
 
+    With burst.exclusive the pods may go only on a burst node: the required node affinity has the burst term alone and nothing is preferred. That suits work too large for the fixed nodes, such as image builds whose Docker daemon needs more memory than they can spare.
+
     Without burst the pods are held by nodeSelector to nodes carrying the eligibility label and the profile's node_selector. With burst the eligibility label moves into a required node affinity that also accepts a node carrying the burst label, so a pod that finds no room on an eligible node stays Pending until a burst node can take it, which is what makes Cluster Autoscaler add one. A preferred term keeps the pods off burst nodes while an eligible node has room. The profile's node_selector still applies to every node, and the tolerations let the pods past the burst nodes' taint.
 
     Args:
@@ -416,6 +423,9 @@ def arc_runner_placement(profile: Mapping[str, Any], eligibility: Mapping[str, A
     if burst is None:
         return {"nodeSelector": {**selector, **profile["node_selector"]}}
     burst_node = {"key": burst["node_label_key"], "operator": "In", "values": burst["node_label_values"]} if burst["node_label_values"] else {"key": burst["node_label_key"], "operator": "Exists"}
+    # An exclusive profile runs on burst nodes only, whatever the eligibility label, so there is no fixed node to prefer.
+    if burst["exclusive"]:
+        return {"nodeSelector": dict(profile["node_selector"]), "affinity": {"nodeAffinity": {"requiredDuringSchedulingIgnoredDuringExecution": {"nodeSelectorTerms": [{"matchExpressions": [burst_node]}]}}}, "tolerations": burst["tolerations"]}
     node_affinity: dict[str, Any] = {"preferredDuringSchedulingIgnoredDuringExecution": [{"weight": _PREFER_FIXED_NODES_WEIGHT, "preference": {"matchExpressions": [{"key": burst["node_label_key"], "operator": "DoesNotExist"}]}}]}
     # With no eligibility label every untainted node is already allowed, and the tolerations alone add the burst nodes.
     if selector:
@@ -448,11 +458,15 @@ def _expand_profile(org_name: str, app_secret: str, index: int, profile: Any, er
     explicit_max_runners = _explicit_max_runners(profile, where, errors)
     burst = _burst(profile, where, errors)
     # Burst runners are added to a ceiling the role derives from the fixed nodes, so they need sizing on a profile that sets its own maxRunners: an explicit max_runners is final, and the autoscaler's usage-driven ceiling has no notion of nodes that do not exist yet.
-    if burst is not None and (sizing_result is None or explicit_max_runners is not None or autoscale):
+    if burst is not None and burst["exclusive"] and (sizing_result is not None or explicit_max_runners is not None or autoscale):
+        errors.append(f"{where}: an exclusive burst takes no sizing, max_runners or autoscale, since its runners use no eligible node and burst.max_runners is the whole ceiling")
+    if burst is not None and not burst["exclusive"] and (sizing_result is None or explicit_max_runners is not None or autoscale):
         errors.append(f"{where}: burst needs sizing, and no max_runners or autoscale, since its max_runners is added to the ceiling sizing derives")
     burst_runners = burst["max_runners"] if burst is not None else 0
     # The static maxRunners the role sets on the release: the profile's own max_runners wins; otherwise sizing supplies it, plus any burst runners, except on the autoscaled profile, whose sizing sets the autoscaler's ceiling rather than its floor. A measured sizing's result is known only at install time, so its max_runners stays None here and the burst runners are added then.
-    if explicit_max_runners is not None:
+    if burst is not None and burst["exclusive"]:
+        max_runners = burst_runners
+    elif explicit_max_runners is not None:
         max_runners = explicit_max_runners
     elif sizing_result is not None and not sizing_result["errors"] and not autoscale:
         max_runners = None if sizing_result["max_runners"] is None else sizing_result["max_runners"] + burst_runners
@@ -494,7 +508,7 @@ def arc_profiles(orgs: Sequence[Any], require_app_id: bool = True) -> dict[str,
 
     Args:
         require_app_id: whether each org must set app_id. The role needs it only when it writes the App Secret; otherwise the App's id is in the existing App Secret, and an app_id that is set is only checked against it.
-        orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes; see arc_runner_placement.
+        orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes, and a profile without sizing may set burst with exclusive true to run on burst nodes only, with burst.max_runners as its maxRunners; see arc_runner_placement.
 
     Returns:
         A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name, suffix, namespace, release, target (namespace/release, joined), app_secret, scale_set_labels, values_file, node_selector, autoscale, sizing, burst (the burst settings with defaults filled in, or None), burst_runners (burst.max_runners, 0 without burst), max_runners (the static maxRunners the role sets, including burst_runners, or None), label and settings, the profile's own keys) and ``autoscaled`` (every profile flagged autoscale, sharing one autoscaler-managed memory pool across their combined scale sets; empty when none are).

roles/github_runner_arc/README.md

@@ -22,7 +22,7 @@ Before touching the cluster the role then checks the secrets it is about to writ
   - `scale_set_labels`: the whole `scaleSetLabels` list, in place of `runs_on_label`. An empty list sets no `scaleSetLabels`, so jobs target the scale set by its release name.
   - `autoscale`: `true` on at most one profile in the whole inventory makes it the scale set the autoscaler manages.
   - `sizing`: derive a runner ceiling from node capacity, given as uniform figures or measured from each eligible node (see Sizing). On an ordinary profile without `max_runners` it sets `maxRunners`; on the autoscaled profile it sets the autoscaler's ceiling, leaving `maxRunners` as the floor.
-  - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`.
+  - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`; with `exclusive: true` the pods run on burst nodes only and `burst.max_runners` is the whole ceiling, with no `sizing`.
 - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from.
 - `github_runner_arc_kubeconfig_path`: the kubeconfig on the host the role runs against. Defaults to `github_runner_cluster`'s kubeconfig; empty means `KUBECONFIG` or `~/.kube/config`.
 - `github_runner_arc_controller_chart_version`, `github_runner_arc_scaleset_chart_version`: chart pins; unset installs the latest chart.
@@ -96,6 +96,8 @@ The runner pods then lose the eligibility `nodeSelector`, and get instead a requ
 
 `burst.max_runners` is added to the ceiling `sizing` derives from the fixed nodes (uniform or measured, safety margin included), so ARC creates more runner pods than the fixed nodes hold; the extra pods stay Pending, which is what makes the autoscaler add a burst node. The safety margin is not applied to it, since burst nodes carry nothing else. Derive it from what the autoscaler may launch: each node group's maximum size times the runners one of its nodes holds (`exadev.github_runner.arc_max_runners` with the node's allocatable figures works that out). The placement comes from the `exadev.github_runner.arc_runner_placement` filter.
 
+`burst.exclusive: true` puts a profile on the burst nodes only: the required node affinity has the burst term alone, with nothing preferred, and the eligibility label no longer applies to its runner pods. It suits work the fixed nodes cannot hold, such as image builds whose Docker daemon needs more memory than those nodes can spare, and pairs with `minRunners: 0` in the profile's values, so no burst node is kept for an idle runner. Such a profile takes no `sizing`, `max_runners` or `autoscale`: `burst.max_runners` is its whole `maxRunners`. Where it shares burst node groups with an overflowing profile, split the groups' capacity between the two `burst.max_runners`, since each is a claim on the same nodes.
+
 ## Image pull Secret from the GitHub App
 
 When the runner image is a private package owned by the org, `github_runner_arc_image_pull_secret_source: app` pulls it with the runners' own GitHub App rather than a registry token tied to a person. The App needs the `packages: read` permission (add it to `github_runner_arc_app_setup_permissions` when creating the App, or to an existing App's settings), approved by the organisation on the App's installation.

roles/github_runner_arc/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 # ARC (Actions Runner Controller) in an existing Kubernetes cluster: the controller, each org's runner scale sets, and the fleet-health platform (heartbeat and autoscaler). Each part is gated on this host's own github_runner_arc_orgs or github_runner_arc_heartbeat_gist_id, so any host that can reach the cluster's API can be the one Ansible installs from: a k3s server host brought up by github_runner_cluster, or localhost with a kubeconfig (playbooks/arc.yml). On a host github_runner_cluster made an agent, this role installs nothing.
 
-# Orgs whose runner scale sets this host installs. Each entry: name, app_id (needed only when the role writes the App Secret; otherwise read from the existing Secret, and checked against it when set), image, private_key (the App's PEM private key, from any source Ansible reads) or private_key_op_reference (an op:// reference the 1Password adapter resolves into private_key), optionally installation_id (resolved from the App when empty), optionally app_secret_name (the App Secret's name in each of the org's namespaces, default <org>-github-app), and scale_set_profiles, a list of {suffix, namespace?, release_name?, values_file?, max_runners?, node_selector?, runs_on_label?, scale_set_labels?, autoscale?, sizing?, burst?}. sizing takes uniform node figures, or measured: true to read each eligible node's free capacity from the cluster at install time. burst ({node_label_key, node_label_values?, tolerations?, max_runners}) lets a sized profile's runner pods overflow onto nodes carrying node_label_key, which a cluster autoscaler adds when pods are Pending, and adds max_runners to the sized maxRunners. Each profile becomes one namespace (namespace, default arc-runners-<org><suffix>), two Secrets in it, and one Helm release (release_name, default <org>-runners<suffix>); setting both overrides lets the role take over scale sets an existing install created under its own names. See the README for every profile key.
+# Orgs whose runner scale sets this host installs. Each entry: name, app_id (needed only when the role writes the App Secret; otherwise read from the existing Secret, and checked against it when set), image, private_key (the App's PEM private key, from any source Ansible reads) or private_key_op_reference (an op:// reference the 1Password adapter resolves into private_key), optionally installation_id (resolved from the App when empty), optionally app_secret_name (the App Secret's name in each of the org's namespaces, default <org>-github-app), and scale_set_profiles, a list of {suffix, namespace?, release_name?, values_file?, max_runners?, node_selector?, runs_on_label?, scale_set_labels?, autoscale?, sizing?, burst?}. sizing takes uniform node figures, or measured: true to read each eligible node's free capacity from the cluster at install time. burst ({node_label_key, node_label_values?, tolerations?, max_runners, exclusive?}) lets a sized profile's runner pods overflow onto nodes carrying node_label_key, which a cluster autoscaler adds when pods are Pending, and adds max_runners to the sized maxRunners; with exclusive: true the pods run on burst nodes only and max_runners is the whole maxRunners. Each profile becomes one namespace (namespace, default arc-runners-<org><suffix>), two Secrets in it, and one Helm release (release_name, default <org>-runners<suffix>); setting both overrides lets the role take over scale sets an existing install created under its own names. See the README for every profile key.
 github_runner_arc_orgs: []
 
 # Directory on the control node that a relative scale_set_profiles[].values_file is read from. Values files are Jinja templates rendered on the control node, so the target host needs no checkout. A profile with no values_file uses the role's own templates/runner-scale-set-values.yaml.j2.

roles/github_runner_arc/meta/argument_specs.yml

@@ -14,7 +14,7 @@ argument_specs:
           - max_runners, when set, is the release's maxRunners and takes precedence over its values file and its sizing.
           - scale_set_labels replaces runs_on_label with the whole list of scaleSetLabels; an empty list sets none, so jobs target the release name.
           - sizing derives a runner ceiling from uniform node figures, or with measured true from each eligible node's allocatable CPU and memory less its current requests and reserve_cpu and reserve_memory_gib.
-          - burst, on a profile with sizing and without max_runners or autoscale, lets its runner pods overflow onto nodes carrying burst.node_label_key (any value, or one of burst.node_label_values), with burst.tolerations for their taint, and adds burst.max_runners to the sized maxRunners.
+          - burst, on a profile with sizing and without max_runners or autoscale, lets its runner pods overflow onto nodes carrying burst.node_label_key (any value, or one of burst.node_label_values), with burst.tolerations for their taint, and adds burst.max_runners to the sized maxRunners. With burst.exclusive true (and no sizing, max_runners or autoscale) the pods run on burst nodes only and burst.max_runners is the whole maxRunners.
       github_runner_arc_values_dir:
         type: str
         default: ""

roles/github_runner_arc/templates/scale-set-overlay.yaml.j2

@@ -1,6 +1,9 @@
 {# The values the role itself owns on every scale-set release, rendered on the control node and merged over the profile's values (nested mappings merge, lists are replaced, as Helm does). With node eligibility, probes, max_runners, sizing and burst all off it holds only the profile's nodeSelector. A measured sizing's result is read from github_runner_arc_measured_sizing, which tasks/measure_sizing.yml fills in. `profile` is the expanded profile from plugins/filter/arc.py. #}
 {% set eligibility = {github_runner_arc_node_label_key: github_runner_arc_node_label_value | string} if github_runner_arc_node_label_key | length > 0 else {} %}
-{% if profile.settings.max_runners is defined and profile.settings.max_runners is not none %}
+{% if profile.burst is not none and profile.burst.exclusive %}
+# Burst nodes only: the runners the burst nodes hold, the profile's burst.max_runners.
+maxRunners: {{ profile.max_runners }}
+{% elif profile.settings.max_runners is defined and profile.settings.max_runners is not none %}
 # The profile's own max_runners, which takes precedence over its values file and its sizing.
 maxRunners: {{ profile.max_runners }}
 {% elif profile.label in (github_runner_arc_measured_sizing | default({})) and not profile.autoscale %}