diff --git a/.cargo/config.toml b/.cargo/config.toml index 0005fc2bd0..8ba7dbaa7e 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -6,3 +6,6 @@ # the header lives in /usr/include/z3/ rather than /usr/include/. The extra -I # is harmless on systems where the path doesn't exist. BINDGEN_EXTRA_CLANG_ARGS = "-I/usr/include/z3" + +[alias] +xtask = "run --package xtask --" diff --git a/Cargo.lock b/Cargo.lock index 76dedf0625..06e9e3a7c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7792,6 +7792,10 @@ dependencies = [ "rustix 1.1.4", ] +[[package]] +name = "xtask" +version = "0.0.0" + [[package]] name = "xterm-color" version = "1.0.2" diff --git a/crates/xtask/Cargo.toml b/crates/xtask/Cargo.toml new file mode 100644 index 0000000000..e049023504 --- /dev/null +++ b/crates/xtask/Cargo.toml @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "xtask" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[dependencies] diff --git a/crates/xtask/scripts/machine/development/centos-stream.sh b/crates/xtask/scripts/machine/development/centos-stream.sh new file mode 100644 index 0000000000..1e901c276d --- /dev/null +++ b/crates/xtask/scripts/machine/development/centos-stream.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +echo "==> Installing OpenShell development dependencies on CentOS Stream" +sudo dnf install -y \ + ca-certificates \ + clang \ + clang-devel \ + cmake \ + curl \ + gcc \ + gcc-c++ \ + git \ + jq \ + make \ + openssh-clients \ + openssl-devel \ + pkgconf-pkg-config \ + python3 \ + rsync \ + socat \ + unzip \ + xz \ + zstd + +z3_version="4.16.0" +case "$(uname -m)" in + aarch64) + z3_asset="z3-${z3_version}-arm64-glibc-2.38.zip" + z3_sha256="87fcd963d3eecb0f12cf1c3ef0ad74e84a3a7bd3caed5d94445645ef94ae6274" + ;; + x86_64) + z3_asset="z3-${z3_version}-x64-glibc-2.39.zip" + z3_sha256="7288c49a5bd6dbafd7b0b0d1f65956b91672da24b08f09242919af159be3418e" + ;; + *) + echo "unsupported architecture for Z3: $(uname -m)" >&2 + exit 1 + ;; +esac + +if pkg-config --atleast-version="${z3_version}" z3 2>/dev/null; then + echo "==> Z3 $(pkg-config --modversion z3) is already installed" +else + echo "==> Installing Z3 ${z3_version} for OpenShell development" + work_dir="$(mktemp -d)" + trap 'rm -rf "${work_dir}"' EXIT + archive="${work_dir}/${z3_asset}" + curl -fL \ + "https://github.com/Z3Prover/z3/releases/download/z3-${z3_version}/${z3_asset}" \ + -o "${archive}" + printf '%s %s\n' "${z3_sha256}" "${archive}" | sha256sum --check --status + unzip -q "${archive}" -d "${work_dir}" + z3_dir="${work_dir}/${z3_asset%.zip}" + + sudo install -D -m 0755 "${z3_dir}/bin/z3" /usr/local/bin/z3 + sudo install -D -m 0755 "${z3_dir}/bin/libz3.so" /usr/local/lib/libz3.so + sudo install -d -m 0755 /usr/local/include + sudo install -m 0644 "${z3_dir}"/include/*.h /usr/local/include/ + sudo install -d -m 0755 /usr/lib64/pkgconfig + sudo tee /usr/lib64/pkgconfig/z3.pc >/dev/null </dev/null + sudo ldconfig + + test "$(pkg-config --modversion z3)" = "${z3_version}" + z3 --version + rm -rf "${work_dir}" + trap - EXIT +fi diff --git a/crates/xtask/scripts/machine/development/common.sh b/crates/xtask/scripts/machine/development/common.sh new file mode 100644 index 0000000000..7c119c4cba --- /dev/null +++ b/crates/xtask/scripts/machine/development/common.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +echo "==> Installing mise" +curl -fsSL https://mise.run | MISE_VERSION=v2026.4.25 sh diff --git a/crates/xtask/scripts/machine/development/ubuntu.sh b/crates/xtask/scripts/machine/development/ubuntu.sh new file mode 100644 index 0000000000..cc1f2c021f --- /dev/null +++ b/crates/xtask/scripts/machine/development/ubuntu.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +echo "==> Installing OpenShell development dependencies on Ubuntu" +sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + git \ + jq \ + libclang-dev \ + libssl-dev \ + libz3-dev \ + musl-tools \ + openssh-client \ + pkg-config \ + python3 \ + python3-venv \ + rsync \ + socat \ + unzip \ + xz-utils \ + zstd diff --git a/crates/xtask/scripts/machine/os/centos-stream.sh b/crates/xtask/scripts/machine/os/centos-stream.sh new file mode 100644 index 0000000000..1942bfdb11 --- /dev/null +++ b/crates/xtask/scripts/machine/os/centos-stream.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +if [ ! -r /etc/os-release ]; then + echo "cannot detect guest OS: /etc/os-release is missing" >&2 + exit 1 +fi + +# shellcheck disable=SC1091 +. /etc/os-release + +if [ "${ID:-}" != "centos" ] || [[ "${VERSION_ID:-}" != 10* ]]; then + echo "expected a CentOS Stream 10 guest, found ${PRETTY_NAME:-unknown}" >&2 + exit 1 +fi + +echo "==> Preparing ${PRETTY_NAME:-CentOS Stream 10}" +sudo dnf install -y \ + audit \ + libselinux-utils \ + policycoreutils + +selinux_mode="$(getenforce)" +echo "==> SELinux mode: ${selinux_mode}" +if [ "${selinux_mode}" != "Enforcing" ]; then + echo "SELinux must be Enforcing for the CentOS Stream 10 e2e target" >&2 + exit 1 +fi + +sudo systemctl enable auditd +if ! sudo systemctl is-active --quiet auditd; then + sudo service auditd start +fi +sudo systemctl is-active --quiet auditd diff --git a/crates/xtask/scripts/machine/os/ubuntu.sh b/crates/xtask/scripts/machine/os/ubuntu.sh new file mode 100644 index 0000000000..f07cea1267 --- /dev/null +++ b/crates/xtask/scripts/machine/os/ubuntu.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +if [ ! -r /etc/os-release ]; then + echo "cannot detect guest OS: /etc/os-release is missing" >&2 + exit 1 +fi + +# shellcheck disable=SC1091 +. /etc/os-release + +if [ "${ID:-}" != "ubuntu" ]; then + echo "expected an Ubuntu guest, found ${ID:-unknown}" >&2 + exit 1 +fi + +echo "==> Preparing Ubuntu ${VERSION_ID:-unknown}" +sudo apt-get update diff --git a/crates/xtask/scripts/machine/suites/podman/centos-stream.sh b/crates/xtask/scripts/machine/suites/podman/centos-stream.sh new file mode 100644 index 0000000000..26fc7e07f5 --- /dev/null +++ b/crates/xtask/scripts/machine/suites/podman/centos-stream.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +echo "==> Installing rootless Podman on CentOS Stream" +sudo dnf install -y podman diff --git a/crates/xtask/scripts/machine/suites/podman/common.sh b/crates/xtask/scripts/machine/suites/podman/common.sh new file mode 100644 index 0000000000..b7a272de57 --- /dev/null +++ b/crates/xtask/scripts/machine/suites/podman/common.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +echo "==> Configuring rootless Podman" +containers_config_dir="${HOME}/.config/containers" +containers_config_drop_in_dir="${containers_config_dir}/containers.conf.d" +mkdir -p "${containers_config_dir}" "${containers_config_drop_in_dir}" +cat >"${containers_config_dir}/containers.conf" <<'EOF' +[containers] +# Use file-backed logs so Docker-compatible log reads work reliably. +log_driver = "k8s-file" + +[engine] +# Use file-backed events so open Libpod streams receive lifecycle events reliably. +events_logger = "file" +EOF +# CentOS Stream ships a containers.conf.d vendor fragment that selects +# journald after the regular user configuration has been merged. A later user +# drop-in overrides it; keeping the regular file also supports older Podman +# releases that do not load per-user drop-ins. +install -m 0644 \ + "${containers_config_dir}/containers.conf" \ + "${containers_config_drop_in_dir}/99-openshell.conf" + +echo "==> Enabling the rootless Podman socket" +sudo loginctl enable-linger "$USER" +systemctl --user daemon-reload +systemctl --user enable --now podman.socket +if ! systemctl --user is-active --quiet podman.socket; then + echo "rootless Podman socket did not become active" >&2 + systemctl --user status --no-pager podman.socket >&2 || true + exit 1 +fi + +require_podman_info() { + local format="$1" + local expected="$2" + local description="$3" + local actual + + if ! actual="$(podman info --format "${format}")"; then + echo "could not inspect Podman ${description}" >&2 + podman info >&2 || true + exit 1 + fi + if [ "${actual}" != "${expected}" ]; then + echo "expected Podman ${description} to be ${expected}, found ${actual:-}" >&2 + exit 1 + fi +} + +require_podman_info '{{.Host.Security.Rootless}}' true "rootless mode" +require_podman_info '{{.Host.LogDriver}}' k8s-file "log driver" +require_podman_info '{{.Host.EventLogger}}' file "event logger" diff --git a/crates/xtask/scripts/machine/suites/podman/ubuntu.sh b/crates/xtask/scripts/machine/suites/podman/ubuntu.sh new file mode 100644 index 0000000000..8cc000a8e1 --- /dev/null +++ b/crates/xtask/scripts/machine/suites/podman/ubuntu.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +echo "==> Installing rootless Podman on Ubuntu" +sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + ca-certificates \ + curl \ + podman diff --git a/crates/xtask/scripts/machine/validation/selinux-audit.sh b/crates/xtask/scripts/machine/validation/selinux-audit.sh new file mode 100644 index 0000000000..4e8c9caaac --- /dev/null +++ b/crates/xtask/scripts/machine/validation/selinux-audit.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +: "${OPENSHELL_SELINUX_AUDIT_START:?OPENSHELL_SELINUX_AUDIT_START must mark the e2e audit window}" + +echo "==> Checking for SELinux AVC denials since ${OPENSHELL_SELINUX_AUDIT_START}" +audit_log="$(mktemp)" +trap 'rm -f "${audit_log}"' EXIT +# The temporary output file is owned by this user; only reading the audit log +# through ausearch requires elevated privileges. +# shellcheck disable=SC2024 +sudo ausearch \ + -m avc,user_avc \ + -ts "${OPENSHELL_SELINUX_AUDIT_START}" \ + --raw >"${audit_log}" 2>/dev/null || true + +if grep -q '^type=' "${audit_log}"; then + echo "SELinux denied one or more operations during the e2e suite:" >&2 + cat "${audit_log}" >&2 + exit 1 +fi + +echo "==> No SELinux AVC denials recorded" diff --git a/crates/xtask/scripts/release-smoke/ubuntu-podman-rootless.sh b/crates/xtask/scripts/release-smoke/ubuntu-podman-rootless.sh new file mode 100644 index 0000000000..0189e54ddb --- /dev/null +++ b/crates/xtask/scripts/release-smoke/ubuntu-podman-rootless.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +# Require xtask to provide the artifact path that it copied onto the machine. +if [ -z "${OPENSHELL_RELEASE_ARTIFACT:-}" ]; then + echo "OPENSHELL_RELEASE_ARTIFACT must be set" >&2 + exit 1 +fi +artifact_path=$OPENSHELL_RELEASE_ARTIFACT +sandbox_name=${OPENSHELL_RELEASE_SMOKE_SANDBOX:-release-smoke-curl} + +# Confirm that this OS+driver script is running on the guest OS it supports. +if [ ! -r /etc/os-release ]; then + echo "cannot detect guest OS: /etc/os-release is missing" >&2 + exit 1 +fi +# shellcheck disable=SC1091 +. /etc/os-release +if [ "${ID:-}" != "ubuntu" ]; then + echo "ubuntu-podman-rootless release smoke test requires Ubuntu, got ${ID:-unknown}" >&2 + exit 1 +fi + +# On failure, collect enough state to diagnose gateway startup or sandbox launch. +# shellcheck disable=SC2154 +trap ' +status=$? +trap - EXIT +if [ "$status" -ne 0 ]; then + echo "==> OpenShell release smoke test diagnostics" >&2 + openshell status >&2 || true + systemctl --user status openshell-gateway --no-pager >&2 || true + systemctl --user status podman.socket --no-pager >&2 || true + journalctl --user -u openshell-gateway --no-pager -n 200 >&2 || true + ss -ltnp >&2 || true + if command -v podman >/dev/null 2>&1; then + podman info >&2 || true + podman ps -a >&2 || true + while IFS= read -r container_id; do + [ -n "$container_id" ] || continue + echo "==> Podman logs: $container_id" >&2 + podman logs "$container_id" >&2 || true + done < <(podman ps -aq) + fi +fi +openshell sandbox delete "$sandbox_name" >/dev/null 2>&1 || true +exit "$status" +' EXIT + +# Install the Ubuntu Debian release artifact under test. +echo "==> Installing Ubuntu release artifact $artifact_path" +sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y "$artifact_path" +openshell --version +openshell-gateway --version + +# Verify that machine setup left the rootless Podman compute driver available. +echo "==> Verifying the rootless Podman compute driver" +systemctl --user is-active --quiet podman.socket +test "$(podman info --format '{{.Host.Security.Rootless}}')" = true + +# Temporary release-test workaround: Debian packages do not yet seed the +# Podman-ready gateway configuration that RPM packages install. Remove this when +# Debian first-start configuration owns the equivalent behavior. +echo "==> Applying the temporary Debian Podman gateway workaround" +mkdir -p "$HOME/.config/openshell" +cat >"$HOME/.config/openshell/gateway.toml" <<'EOF' +[openshell] +version = 1 + +[openshell.gateway] +bind_address = "0.0.0.0:17670" +compute_drivers = ["podman"] +EOF + +# Start the packaged gateway service and register it with the CLI. +echo "==> Starting the packaged OpenShell gateway" +systemctl --user enable --now openshell-gateway +systemctl --user is-active --quiet openshell-gateway + +echo "==> Registering the packaged gateway" +openshell gateway add https://127.0.0.1:17670 --local --name openshell + +# Wait until the CLI can reach the packaged gateway. +echo "==> Waiting for the packaged gateway" +status_output="" +for _ in $(seq 1 30); do + if status_output="$(NO_COLOR=1 openshell status 2>&1)" && grep -q "Version:" <<<"$status_output"; then + break + fi + sleep 1 +done +if ! grep -q "Version:" <<<"$status_output"; then + echo "openshell status did not report a connected gateway:" >&2 + echo "$status_output" >&2 + exit 1 +fi + +# Create a sandbox and verify that the default policy denies undeclared egress. +echo "==> Creating a sandbox and verifying default-deny networking" +timeout 600 openshell sandbox create \ + --name "$sandbox_name" \ + --no-tty \ + -- \ + sh -c ' + command -v curl >/dev/null + if curl --fail --show-error --silent --max-time 20 https://example.com; then + echo "curl unexpectedly succeeded without a network policy" >&2 + exit 1 + fi + echo "curl was denied as expected" + ' + +# Report the single result this smoke test is meant to prove. +echo "==> Ubuntu rootless Podman release smoke test passed: sandbox curl was denied as expected" diff --git a/crates/xtask/scripts/sync-development-source.sh b/crates/xtask/scripts/sync-development-source.sh new file mode 100644 index 0000000000..5aed9bb0cc --- /dev/null +++ b/crates/xtask/scripts/sync-development-source.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +: "${OPENSHELL_SOURCE_MOUNT:?OPENSHELL_SOURCE_MOUNT must name the mounted host checkout}" +: "${OPENSHELL_GUEST_SOURCE_DIR:?OPENSHELL_GUEST_SOURCE_DIR must name the guest checkout}" +: "${OPENSHELL_CACHE_DISK:?OPENSHELL_CACHE_DISK must name the development cache directory}" +: "${CARGO_TARGET_DIR:?CARGO_TARGET_DIR must name the Cargo target directory}" + +if [ ! -f "${OPENSHELL_SOURCE_MOUNT}/mise.toml" ]; then + echo "mounted OpenShell checkout is missing mise.toml: ${OPENSHELL_SOURCE_MOUNT}" >&2 + exit 1 +fi + +echo "==> Syncing the mounted OpenShell checkout onto the machine" +mkdir -p "${OPENSHELL_GUEST_SOURCE_DIR}" +rsync -a --delete \ + --exclude='/.cache/' \ + --exclude='/.env' \ + --exclude='/.git/' \ + --exclude='/.jj/' \ + --exclude='/.venv/' \ + --exclude='/e2e/rust/target/' \ + --exclude='/kubeconfig' \ + --exclude='/target/' \ + "${OPENSHELL_SOURCE_MOUNT}/" \ + "${OPENSHELL_GUEST_SOURCE_DIR}/" + +mkdir -p "${OPENSHELL_GUEST_SOURCE_DIR}/.cache" +if [ "${OPENSHELL_CACHE_DISK}" = "${OPENSHELL_GUEST_SOURCE_DIR}/.cache" ]; then + mkdir -p "${OPENSHELL_GUEST_SOURCE_DIR}/.cache/sccache" "${CARGO_TARGET_DIR}" +else + echo "==> Preparing the persistent Lima development cache disk" + sudo install -d -o "${USER}" -g "$(id -gn)" "${OPENSHELL_CACHE_DISK}" + mkdir -p "${OPENSHELL_CACHE_DISK}/sccache" "${CARGO_TARGET_DIR}" + rm -rf "${OPENSHELL_GUEST_SOURCE_DIR}/.cache/sccache" + ln -s "${OPENSHELL_CACHE_DISK}/sccache" \ + "${OPENSHELL_GUEST_SOURCE_DIR}/.cache/sccache" + rm -rf "${OPENSHELL_GUEST_SOURCE_DIR}/target" + ln -s "${CARGO_TARGET_DIR}" "${OPENSHELL_GUEST_SOURCE_DIR}/target" +fi diff --git a/crates/xtask/src/e2e.rs b/crates/xtask/src/e2e.rs new file mode 100644 index 0000000000..74afb10aec --- /dev/null +++ b/crates/xtask/src/e2e.rs @@ -0,0 +1,273 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::{OsStr, OsString}; +use std::process::{Command, ExitCode, ExitStatus}; + +use crate::machine::{MachineOptions, MachineOptionsBuilder}; +use crate::platform::parse_machine_os; +use crate::tasks::{TaskResult, exit_code, print_help_if_requested}; + +const HELP: &str = "Run an e2e suite locally or on a target machine. + +Usage: + cargo xtask e2e --suite [--test ] [--os ] [--provider ] [--arch ] [--snapshot] [--rebuild-machine] [--keep-machine] + +Defaults for machine execution: + --provider lima + --os ubuntu-26.04 when --provider is supplied"; + +pub fn run(args: impl Iterator) -> TaskResult { + let mut args = args.peekable(); + if print_help_if_requested(&mut args, HELP) { + return Ok(ExitCode::SUCCESS); + } + + let command = E2eCommand::parse(args)?; + let status = match command.machine { + Some(options) => crate::e2e_machine::run(&command.selection, &options), + None => run_local(&command.selection), + }?; + Ok(exit_code(status)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum E2eSuite { + Podman, +} + +impl E2eSuite { + pub(crate) const fn id(self) -> &'static str { + match self { + Self::Podman => "podman", + } + } + + fn parse(value: &OsStr) -> Result { + match value.to_str() { + Some("podman") => Ok(Self::Podman), + Some(value) => Err(format!("unsupported e2e suite: {value} (expected podman)")), + None => Err("--suite must be valid UTF-8".to_owned()), + } + } + + const fn script(self) -> &'static str { + match self { + Self::Podman => "e2e/rust/e2e-podman.sh", + } + } + + const fn test_environment(self) -> &'static str { + match self { + Self::Podman => "OPENSHELL_E2E_PODMAN_TEST", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct E2eSelection { + pub(crate) suite: E2eSuite, + pub(crate) test: Option, +} + +impl E2eSelection { + pub(crate) const fn suite(&self) -> E2eSuite { + self.suite + } + + pub(crate) fn test(&self) -> Option<&str> { + self.test.as_deref() + } +} + +struct E2eCommand { + selection: E2eSelection, + machine: Option, +} + +impl E2eCommand { + fn parse(mut args: impl Iterator) -> Result { + let mut suite = None; + let mut os = None; + let mut test = None; + let mut machine = MachineOptionsBuilder::default(); + + while let Some(argument) = args.next() { + match argument.to_str() { + Some("--suite") => { + let value = args + .next() + .ok_or_else(|| "--suite requires a value".to_owned())?; + if suite.replace(E2eSuite::parse(&value)?).is_some() { + return Err("--suite may only be specified once".to_owned()); + } + } + Some("--test") => { + let value = args + .next() + .ok_or_else(|| "--test requires a value".to_owned())?; + let value = value + .into_string() + .map_err(|_| "--test must be valid UTF-8".to_owned())?; + if value.is_empty() { + return Err("--test may not be empty".to_owned()); + } + if test.replace(value).is_some() { + return Err("--test may only be specified once".to_owned()); + } + } + Some("--os") => { + let value = args + .next() + .ok_or_else(|| "--os requires a value".to_owned())?; + if os.replace(parse_machine_os(&value, "--os")?).is_some() { + return Err("--os may only be specified once".to_owned()); + } + } + Some(value) if machine.parse_argument(value, &mut args)? => {} + Some(value) => return Err(format!("unknown e2e option: {value}")), + None => return Err("e2e options must be valid UTF-8".to_owned()), + } + } + + Ok(Self { + selection: E2eSelection { + suite: suite.ok_or_else(|| "e2e requires --suite ".to_owned())?, + test, + }, + machine: machine.finish(os)?, + }) + } +} + +fn run_local(selection: &E2eSelection) -> Result { + local_command(selection).status().map_err(|error| { + format!( + "failed to run the {} e2e suite: {error}", + selection.suite.id() + ) + }) +} + +fn local_command(selection: &E2eSelection) -> Command { + let mut command = Command::new("mise"); + command.args(["exec", "--", selection.suite.script()]); + if let Some(test) = selection.test() { + command.env(selection.suite.test_environment(), test); + } + command +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::machine::Provider; + use crate::platform::{Arch, MachineOs}; + + #[test] + fn parses_a_local_e2e_selection() { + let command = E2eCommand::parse( + ["--suite", "podman", "--test", "smoke"] + .into_iter() + .map(OsString::from), + ) + .expect("local e2e command should parse"); + + assert_eq!(command.selection.suite, E2eSuite::Podman); + assert_eq!(command.selection.test.as_deref(), Some("smoke")); + assert!(command.machine.is_none()); + } + + #[test] + fn parses_a_machine_e2e_selection() { + let command = E2eCommand::parse( + [ + "--suite", + "podman", + "--test", + "smoke", + "--os", + "ubuntu-24.04", + "--provider", + "lima", + "--arch", + "arm64", + "--keep-machine", + "--rebuild-machine", + "--snapshot", + ] + .into_iter() + .map(OsString::from), + ) + .expect("machine e2e command should parse"); + + let machine = command.machine.expect("machine options should be present"); + assert_eq!(machine.os, MachineOs::Ubuntu24_04); + assert_eq!(machine.arch, Some(Arch::Arm64)); + assert_eq!(machine.provider, Provider::Lima); + assert!(machine.keep); + assert!(machine.rebuild); + assert!(machine.snapshot); + } + + #[test] + fn parses_a_centos_stream_machine_selection() { + let command = E2eCommand::parse( + ["--suite", "podman", "--os", "centos-stream-10"] + .into_iter() + .map(OsString::from), + ) + .expect("CentOS Stream e2e command should parse"); + + let machine = command.machine.expect("an OS should request a machine"); + assert_eq!(machine.os, MachineOs::CentosStream10); + assert_eq!(machine.provider, Provider::Lima); + } + + #[test] + fn rejects_machine_options_without_an_os_or_provider() { + let error = E2eCommand::parse( + ["--suite", "podman", "--snapshot"] + .into_iter() + .map(OsString::from), + ) + .err() + .expect("machine-only options should require an OS or provider"); + + assert!(error.contains("--snapshot requires --os or --provider")); + } + + #[test] + fn a_provider_uses_the_default_os() { + let command = E2eCommand::parse( + ["--suite", "podman", "--provider", "lima"] + .into_iter() + .map(OsString::from), + ) + .expect("a provider should select the default machine OS"); + + let machine = command + .machine + .expect("a provider should request a machine"); + assert_eq!(machine.provider, Provider::Lima); + assert_eq!(machine.os, MachineOs::Ubuntu26_04); + } + + #[test] + fn builds_the_direct_suite_command() { + let selection = E2eSelection { + suite: E2eSuite::Podman, + test: Some("smoke".to_owned()), + }; + let command = local_command(&selection); + + assert_eq!(command.get_program(), "mise"); + assert_eq!( + command.get_args().collect::>(), + ["exec", "--", "e2e/rust/e2e-podman.sh"] + ); + assert!(command.get_envs().any(|(name, value)| { + name == "OPENSHELL_E2E_PODMAN_TEST" && value == Some(OsStr::new("smoke")) + })); + } +} diff --git a/crates/xtask/src/e2e_machine.rs b/crates/xtask/src/e2e_machine.rs new file mode 100644 index 0000000000..6d360a5f2d --- /dev/null +++ b/crates/xtask/src/e2e_machine.rs @@ -0,0 +1,386 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::env; +use std::path::{Path, PathBuf}; +use std::process::ExitStatus; + +use crate::e2e::{E2eSelection, E2eSuite}; +use crate::machine::{ + HostMount, MachineOptions, MachineProvider, MachineRequest, PersistentDisk, path_hash, + stable_hash, +}; +use crate::platform::{Arch, MachineOs, OsFamily}; + +const CENTOS_STREAM_OS_SETUP: &str = include_str!("../scripts/machine/os/centos-stream.sh"); +const UBUNTU_OS_SETUP: &str = include_str!("../scripts/machine/os/ubuntu.sh"); +const CENTOS_STREAM_DEVELOPMENT_SETUP: &str = + include_str!("../scripts/machine/development/centos-stream.sh"); +const UBUNTU_DEVELOPMENT_SETUP: &str = include_str!("../scripts/machine/development/ubuntu.sh"); +const COMMON_DEVELOPMENT_SETUP: &str = include_str!("../scripts/machine/development/common.sh"); +const CENTOS_STREAM_PODMAN_SETUP: &str = + include_str!("../scripts/machine/suites/podman/centos-stream.sh"); +const UBUNTU_PODMAN_SETUP: &str = include_str!("../scripts/machine/suites/podman/ubuntu.sh"); +const COMMON_PODMAN_SETUP: &str = include_str!("../scripts/machine/suites/podman/common.sh"); +const SELINUX_AUDIT_VALIDATION: &str = + include_str!("../scripts/machine/validation/selinux-audit.sh"); +const SYNC_DEVELOPMENT_SOURCE: &str = include_str!("../scripts/sync-development-source.sh"); +const MISE_TOML: &[u8] = include_bytes!("../../../mise.toml"); +const MISE_LOCK: &[u8] = include_bytes!("../../../mise.lock"); +const GUEST_SOURCE_DIR: &str = "${HOME}/.local/share/openshell-e2e/source"; + +fn host_arch() -> Result { + match env::consts::ARCH { + "x86_64" => Ok(Arch::Amd64), + "aarch64" => Ok(Arch::Arm64), + arch => Err(format!( + "unsupported host architecture: {arch} (pass --arch amd64 or --arch arm64)" + )), + } +} + +pub(crate) fn run( + selection: &E2eSelection, + options: &MachineOptions, +) -> Result { + run_with_provider(&options.provider, selection, options) +} + +fn run_with_provider( + provider: &P, + selection: &E2eSelection, + options: &MachineOptions, +) -> Result { + let source = project_root().canonicalize().map_err(|error| { + format!( + "cannot resolve the OpenShell checkout at {}: {error}", + project_root().display() + ) + })?; + if !source.join("mise.toml").is_file() { + return Err(format!( + "OpenShell checkout is missing mise.toml: {}", + source.display() + )); + } + + let arch = options.arch.map_or_else(host_arch, Ok)?; + let cache_disk = options + .snapshot + .then(|| development_cache_disk(options.os, arch, selection.suite(), &source)); + let cache_mount = cache_disk + .as_ref() + .map(|disk| provider.persistent_disk_mount_point(disk)); + let setup_script = development_setup_script( + options.os, + selection.suite(), + &source, + cache_mount.as_deref(), + )?; + let test_script = e2e_guest_script(selection, options.os, &source, cache_mount.as_deref())?; + let machine = provider.acquire( + MachineRequest { + os: options.os, + arch, + purpose: "e2e", + profile: selection.suite().id(), + forward_env: vec!["MISE_GITHUB_TOKEN"], + keep_on_failure: options.keep, + reuse: options.reuse_policy(stable_hash(&[MISE_TOML, MISE_LOCK])), + host_mounts: vec![HostMount::read_only(&source)], + persistent_disks: cache_disk.into_iter().collect(), + }, + &setup_script, + )?; + + println!( + "==> Running {} e2e suite on {} with {}", + selection.suite().id(), + options.os.id(), + machine.name() + ); + let test_result = machine.run_script(&test_script, "e2e test"); + + if options.keep { + eprintln!("Machine kept for inspection: {}", machine.name()); + return test_result; + } + + let release_result = machine.release(); + match (test_result, release_result) { + (Ok(status), Ok(())) => Ok(status), + (Err(error), Ok(())) => Err(error), + (Err(error), Err(release_error)) => { + Err(format!("{error}; release also failed: {release_error}")) + } + (Ok(status), Err(error)) if status.success() => Err(error), + (Ok(status), Err(error)) => { + eprintln!("warning: {error}"); + Ok(status) + } + } +} + +fn project_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + +fn development_cache_disk( + os: MachineOs, + arch: Arch, + suite: E2eSuite, + source: &Path, +) -> PersistentDisk { + let arch = match arch { + Arch::Amd64 => "x64", + Arch::Arm64 => "a64", + }; + PersistentDisk::ext4( + format!( + "os-e2e-cache-{}-{}-{arch}-{:08x}", + os.short_name(), + suite.id(), + path_hash(source) + ), + "50GiB", + ) +} + +fn development_setup_script( + os: MachineOs, + suite: E2eSuite, + source: &Path, + cache_mount: Option<&str>, +) -> Result { + let mut scripts = os_setup_scripts(os); + scripts.extend(development_setup_scripts(os)); + scripts.extend(suite_setup_scripts(suite, os)); + scripts.push(COMMON_DEVELOPMENT_SETUP); + scripts.push(SYNC_DEVELOPMENT_SOURCE); + + Ok(format!( + "{}\n{}\n\ + export PATH=\"${{HOME}}/.local/bin:${{HOME}}/.local/share/mise/shims:${{PATH}}\"\n\ + cd \"${{OPENSHELL_GUEST_SOURCE_DIR}}\"\n\ + mise trust mise.toml\n\ + mise install --locked\n\ + mise reshim\n", + guest_environment(source, cache_mount)?, + scripts.join("\n") + )) +} + +fn os_setup_scripts(os: MachineOs) -> Vec<&'static str> { + match os.family() { + OsFamily::CentosStream => vec![CENTOS_STREAM_OS_SETUP], + OsFamily::Ubuntu => vec![UBUNTU_OS_SETUP], + } +} + +fn development_setup_scripts(os: MachineOs) -> Vec<&'static str> { + match os.family() { + OsFamily::CentosStream => vec![CENTOS_STREAM_DEVELOPMENT_SETUP], + OsFamily::Ubuntu => vec![UBUNTU_DEVELOPMENT_SETUP], + } +} + +fn suite_setup_scripts(suite: E2eSuite, os: MachineOs) -> Vec<&'static str> { + match (suite, os.family()) { + (E2eSuite::Podman, OsFamily::CentosStream) => { + vec![CENTOS_STREAM_PODMAN_SETUP, COMMON_PODMAN_SETUP] + } + (E2eSuite::Podman, OsFamily::Ubuntu) => { + vec![UBUNTU_PODMAN_SETUP, COMMON_PODMAN_SETUP] + } + } +} + +fn e2e_guest_script( + selection: &E2eSelection, + os: MachineOs, + source: &Path, + cache_mount: Option<&str>, +) -> Result { + let test = selection + .test() + .map_or_else(String::new, |test| format!(" --test {}", shell_quote(test))); + let command = format!( + "cargo xtask e2e --suite {}{test}", + shell_quote(selection.suite().id()) + ); + let invocation = match os.family() { + OsFamily::CentosStream => format!( + "export OPENSHELL_SELINUX_AUDIT_START=\"$(date -u '+%m/%d/%Y %H:%M:%S')\"\n\ + set +e\n\ + {command}\n\ + e2e_status=$?\n\ + set -e\n\ + {SELINUX_AUDIT_VALIDATION}\n\ + exit \"${{e2e_status}}\"" + ), + OsFamily::Ubuntu => format!("exec {command}"), + }; + Ok(format!( + "{}\n\ + {SYNC_DEVELOPMENT_SOURCE}\n\ + export PATH=\"${{HOME}}/.local/bin:${{HOME}}/.local/share/mise/shims:${{PATH}}\"\n\ + cd \"${{OPENSHELL_GUEST_SOURCE_DIR}}\"\n\ + mise trust mise.toml\n\ + mise install --locked\n\ + {invocation}\n", + guest_environment(source, cache_mount)?, + )) +} + +fn guest_environment(source: &Path, cache_mount: Option<&str>) -> Result { + let cache_environment = if let Some(mount_point) = cache_mount { + format!( + "export OPENSHELL_CACHE_DISK='{}'\n\ + export CARGO_TARGET_DIR=\"${{OPENSHELL_CACHE_DISK}}/cargo-target\"", + mount_point + ) + } else { + "export OPENSHELL_CACHE_DISK=\"${OPENSHELL_GUEST_SOURCE_DIR}/.cache\"\n\ + export CARGO_TARGET_DIR=\"${OPENSHELL_GUEST_SOURCE_DIR}/target\"" + .to_owned() + }; + Ok(format!( + "export OPENSHELL_SOURCE_MOUNT={}\n\ + export OPENSHELL_GUEST_SOURCE_DIR=\"{GUEST_SOURCE_DIR}\"\n\ + {cache_environment}", + shell_quote_path(source)? + )) +} + +fn shell_quote_path(path: &Path) -> Result { + let value = path + .to_str() + .ok_or_else(|| format!("source path is not valid UTF-8: {}", path.display()))?; + Ok(shell_quote(value)) +} + +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\"'\"'")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::machine::Provider; + + #[test] + fn composes_setup_and_delegates_to_the_local_runner() { + let source = Path::new("/Volumes/work/OpenShell"); + let cache_disk = development_cache_disk( + MachineOs::Ubuntu24_04, + Arch::Arm64, + E2eSuite::Podman, + source, + ); + assert_eq!(cache_disk.size, "50GiB"); + assert_eq!(cache_disk.filesystem, "ext4"); + assert_eq!( + cache_disk.name, + "os-e2e-cache-ubuntu2404-podman-a64-7e4fa9a2" + ); + let cache_mount = Provider::Lima.persistent_disk_mount_point(&cache_disk); + + let setup = development_setup_script( + MachineOs::Ubuntu24_04, + E2eSuite::Podman, + source, + Some(&cache_mount), + ) + .expect("setup should be supported"); + let os_index = setup.find("Preparing Ubuntu").expect("OS setup"); + let development_index = setup + .find("Installing OpenShell development dependencies") + .expect("development setup"); + let podman_index = setup + .find("Installing rootless Podman") + .expect("Podman setup"); + let podman_activation_index = setup + .find("Enabling the rootless Podman socket") + .expect("Podman common setup"); + assert!(setup.contains("log_driver = \"k8s-file\"")); + assert!(setup.contains("events_logger = \"file\"")); + let mise_index = setup.find("Installing mise").expect("mise setup"); + assert!(os_index < development_index); + assert!(development_index < podman_index); + assert!(podman_index < podman_activation_index); + assert!(podman_activation_index < mise_index); + + let selection = E2eSelection { + suite: E2eSuite::Podman, + test: Some("smoke".to_owned()), + }; + let test = e2e_guest_script( + &selection, + MachineOs::Ubuntu24_04, + source, + Some(&cache_mount), + ) + .expect("test should be supported"); + assert!(test.contains("exec cargo xtask e2e --suite 'podman' --test 'smoke'")); + assert!(!test.contains("OPENSHELL_E2E_PODMAN_TEST")); + assert!(test.contains("export OPENSHELL_SOURCE_MOUNT='/Volumes/work/OpenShell'")); + assert!(test.contains( + "export OPENSHELL_CACHE_DISK='/mnt/lima-os-e2e-cache-ubuntu2404-podman-a64-7e4fa9a2'" + )); + } + + #[test] + fn composes_a_podman_only_centos_stream_selinux_target() { + let source = Path::new("/Volumes/work/OpenShell"); + let setup = + development_setup_script(MachineOs::CentosStream10, E2eSuite::Podman, source, None) + .expect("CentOS Stream setup should be supported"); + + let os_index = setup + .find("Preparing ${PRETTY_NAME:-CentOS Stream 10}") + .expect("CentOS Stream OS setup"); + let development_index = setup + .find("Installing OpenShell development dependencies on CentOS Stream") + .expect("CentOS Stream development setup"); + let z3_index = setup + .find("Installing Z3 ${z3_version} for OpenShell development") + .expect("CentOS Stream Z3 setup"); + let podman_index = setup + .find("Installing rootless Podman on CentOS Stream") + .expect("CentOS Stream Podman setup"); + let podman_activation_index = setup + .find("Enabling the rootless Podman socket") + .expect("common Podman setup"); + assert!(os_index < development_index); + assert!(development_index < z3_index); + assert!(z3_index < podman_index); + assert!(podman_index < podman_activation_index); + assert!(setup.contains("SELinux must be Enforcing")); + assert!(!setup.contains("z3-devel")); + assert!(!setup.contains("dockerd")); + assert!(!setup.contains("docker.service")); + assert!(setup.contains("containers.conf.d")); + assert!(setup.contains("99-openshell.conf")); + + let selection = E2eSelection { + suite: E2eSuite::Podman, + test: None, + }; + let test = e2e_guest_script(&selection, MachineOs::CentosStream10, source, None) + .expect("CentOS Stream test should be supported"); + assert!(test.contains("cargo xtask e2e --suite 'podman'")); + assert!(!test.contains("exec cargo xtask e2e")); + assert!(test.contains("OPENSHELL_SELINUX_AUDIT_START")); + assert!(test.contains("ausearch")); + assert!(test.contains("exit \"${e2e_status}\"")); + } + + #[test] + fn quotes_source_paths_and_test_names_for_the_guest_shell() { + assert_eq!( + shell_quote_path(Path::new("/tmp/source'checkout")).expect("path should quote"), + "'/tmp/source'\"'\"'checkout'" + ); + assert_eq!(shell_quote("test'name"), "'test'\"'\"'name'"); + } +} diff --git a/crates/xtask/src/lima.rs b/crates/xtask/src/lima.rs new file mode 100644 index 0000000000..47127c6d9d --- /dev/null +++ b/crates/xtask/src/lima.rs @@ -0,0 +1,862 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::OsString; +use std::io::Write; +use std::path::Path; +use std::process::{Command, ExitStatus, Stdio}; + +use crate::machine::{ + HostMount, Machine, MachineProvider, MachineRequest, PersistentDisk, hash_request_resources, + stable_hash, +}; +use crate::platform::{Arch, MachineOs}; + +const BASE_SNAPSHOT_VERSION: &str = "base-v5"; + +#[derive(Debug, Clone)] +struct AttachedDisk { + request: PersistentDisk, + format: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Backend { + Default, + Qemu, +} + +impl Backend { + const fn name(self) -> &'static str { + match self { + Self::Default => "def", + Self::Qemu => "qemu", + } + } + + const fn lima(self) -> Option<&'static str> { + match self { + Self::Default => None, + Self::Qemu => Some("qemu"), + } + } +} + +fn instance_name(request: &MachineRequest, backend: Backend, ephemeral: bool) -> String { + let arch = match request.arch { + Arch::Amd64 => "amd64", + Arch::Arm64 => "arm64", + }; + let pid = ephemeral.then(|| std::process::id().to_string()); + let hash = stable_hash(&[ + request.os.id().as_bytes(), + request.purpose.as_bytes(), + request.profile.as_bytes(), + arch.as_bytes(), + backend.name().as_bytes(), + pid.as_deref().unwrap_or_default().as_bytes(), + ]); + let hash = hash_request_resources(hash, request); + format!( + "openshell-{}-{}-{hash:08x}", + request.purpose, + request.os.id() + ) +} + +fn lima_template(os: MachineOs) -> &'static str { + match os { + MachineOs::CentosStream10 => "template:centos-stream-10", + MachineOs::Ubuntu24_04 => "template:ubuntu-24.04", + MachineOs::Ubuntu26_04 => "template:ubuntu-26.04", + } +} + +fn snapshot_tag(request: &MachineRequest, setup_script: &str) -> String { + let preparation_key = request.reuse.preparation_key().to_le_bytes(); + let template = lima_template(request.os); + let hash = stable_hash(&[ + request.os.id().as_bytes(), + template.as_bytes(), + request.profile.as_bytes(), + setup_script.as_bytes(), + &preparation_key, + ]); + let hash = hash_request_resources(hash, request); + format!("{BASE_SNAPSHOT_VERSION}-{hash:08x}") +} + +pub struct LimaProvider; + +struct LimaMachine { + name: String, + reusable: bool, + forward_env: Vec<&'static str>, +} + +impl MachineProvider for LimaProvider { + fn persistent_disk_mount_point(&self, disk: &PersistentDisk) -> String { + format!("/mnt/lima-{}", disk.name) + } + + fn acquire( + &self, + request: MachineRequest, + setup_script: &str, + ) -> Result, String> { + checked( + Command::new("limactl").arg("--version"), + "check the Lima installation", + )?; + let disks = ensure_disks(&request.persistent_disks)?; + + let qemu_available = request.reuse.enabled() && driver_available("qemu")?; + let reusable = request.reuse.enabled() && qemu_available; + if request.reuse.enabled() && !qemu_available { + eprintln!( + "warning: QEMU is not available; using Lima's default backend without snapshots" + ); + } + + let backend = if reusable { + Backend::Qemu + } else { + Backend::Default + }; + let name = instance_name(&request, backend, !reusable); + + prepare_instance(&name, &request, &disks, backend, reusable, setup_script)?; + Ok(Box::new(LimaMachine { + name, + reusable, + forward_env: request.forward_env, + })) + } +} + +impl Machine for LimaMachine { + fn name(&self) -> &str { + &self.name + } + + fn copy_file(&self, source: &Path, destination: &str) -> Result<(), String> { + let guest_path = format!("{}:{destination}", self.name); + checked( + Command::new("limactl") + .args(["--tty=false", "copy", "--backend=scp"]) + .arg(source) + .arg(guest_path), + "copy a file into Lima", + ) + } + + fn run_script(&self, script: &str, description: &str) -> Result { + run_guest_script(&self.name, script, description, &self.forward_env) + } + + fn release(&self) -> Result<(), String> { + if self.reusable { + stop_instance(&self.name) + } else { + delete_instance(&self.name, "delete the Lima test instance") + } + } +} + +fn prepare_instance( + instance: &str, + request: &MachineRequest, + disks: &[AttachedDisk], + backend: Backend, + use_snapshot: bool, + setup_script: &str, +) -> Result<(), String> { + let snapshot = snapshot_tag(request, setup_script); + let status = instance_status(instance)?; + let can_restore = use_snapshot + && !request.reuse.rebuild() + && matches!(status.as_deref(), Some("Running" | "Stopped")) + && snapshot_exists(instance, &snapshot)?; + + if can_restore { + let result = restore_prepared_instance(instance, &snapshot); + return finish_preparation( + result, + instance, + request.keep_on_failure, + &format!("Stopping Lima instance {instance} after failed snapshot restore"), + || stop_instance(instance), + ); + } + + if status.is_some() { + println!("==> Rebuilding Lima instance {instance}"); + delete_instance(instance, "delete the stale Lima test instance")?; + } + + let result = prepare_new_instance( + instance, + request, + disks, + backend, + use_snapshot, + setup_script, + &snapshot, + ); + finish_preparation( + result, + instance, + request.keep_on_failure, + &format!("Removing incomplete Lima instance {instance}"), + || delete_instance_if_exists(instance), + ) +} + +fn restore_prepared_instance(instance: &str, snapshot: &str) -> Result<(), String> { + stop_instance(instance)?; + println!("==> Restoring Lima snapshot {snapshot}"); + checked( + Command::new("limactl") + .args(["--tty=false", "snapshot", "apply", instance, "--tag"]) + .arg(snapshot), + "restore the Lima test snapshot", + )?; + checked( + Command::new("limactl").args(["--tty=false", "start", instance]), + "start the prepared Lima test instance", + ) +} + +fn prepare_new_instance( + instance: &str, + request: &MachineRequest, + disks: &[AttachedDisk], + backend: Backend, + use_snapshot: bool, + setup_script: &str, + snapshot: &str, +) -> Result<(), String> { + start_new_instance(instance, request, disks, backend)?; + let setup_status = run_guest_script(instance, setup_script, "VM setup", &request.forward_env)?; + if !setup_status.success() { + return Err(format!( + "VM setup failed with exit code {}", + display_exit_code(setup_status) + )); + } + + if !use_snapshot { + return Ok(()); + } + + stop_instance(instance)?; + disable_disk_format(instance, disks)?; + println!("==> Creating Lima snapshot {snapshot}"); + checked( + Command::new("limactl") + .args(["--tty=false", "snapshot", "create", instance, "--tag"]) + .arg(snapshot), + "create the Lima test snapshot", + )?; + checked( + Command::new("limactl").args(["--tty=false", "start", instance]), + "start the prepared Lima test instance", + ) +} + +fn finish_preparation( + result: Result, + instance: &str, + keep_on_failure: bool, + description: &str, + cleanup: impl FnOnce() -> Result<(), String>, +) -> Result { + match result { + Ok(value) => Ok(value), + Err(error) if keep_on_failure => { + eprintln!("VM kept for inspection after provisioning failure: {instance}"); + Err(error) + } + Err(error) => { + eprintln!("==> {description}"); + match cleanup() { + Ok(()) => Err(error), + Err(cleanup_error) => Err(format!("{error}; cleanup also failed: {cleanup_error}")), + } + } + } +} + +fn delete_instance_if_exists(instance: &str) -> Result<(), String> { + if instance_status(instance)?.is_none() { + return Ok(()); + } + delete_instance(instance, "delete the incomplete Lima test instance") +} + +fn start_new_instance( + instance: &str, + request: &MachineRequest, + disks: &[AttachedDisk], + backend: Backend, +) -> Result<(), String> { + println!("==> Creating Lima instance {instance}"); + let arch = match request.arch { + Arch::Amd64 => "x86_64", + Arch::Arm64 => "aarch64", + }; + let mut process = Command::new("limactl"); + process.args([ + "--tty=false", + "start", + "--name", + instance, + "--arch", + arch, + "--cpus", + "4", + "--memory", + "8", + "--disk", + "30", + ]); + if let Some(vm_type) = backend.lima() { + process.args(["--vm-type", vm_type]); + } + configure_mount_mode(&mut process, &request.host_mounts); + configure_disks(&mut process, disks)?; + process.arg(lima_template(request.os)); + checked(&mut process, "create the Lima test instance") +} + +fn configure_mount_mode(process: &mut Command, host_mounts: &[HostMount]) { + if host_mounts.is_empty() { + process.arg("--plain"); + } else { + // Plain mode disables the guest agent and therefore ignores host mounts. + // Keep Lima-managed containerd disabled while enabling only the + // explicitly requested development host mounts. + process.args(["--containerd", "none"]); + for mount in host_mounts { + process.arg("--mount-only").arg(mount_argument(mount)); + } + } +} + +fn mount_argument(mount: &HostMount) -> OsString { + let mut argument = mount.path.as_os_str().to_os_string(); + if mount.writable { + argument.push(":w"); + } + argument +} + +fn ensure_disks(disks: &[PersistentDisk]) -> Result, String> { + if disks.is_empty() { + return Ok(Vec::new()); + } + + let output = Command::new("limactl") + .args(["--tty=false", "disk", "list", "--json"]) + .output() + .map_err(|error| format!("failed to list Lima disks: {error}"))?; + if !output.status.success() { + return Err("failed to list Lima disks".to_owned()); + } + + let mut attached = Vec::with_capacity(disks.len()); + for disk in disks { + validate_disk_name(&disk.name)?; + if disk_list_contains(&output.stdout, &disk.name) { + attached.push(AttachedDisk { + request: disk.clone(), + format: false, + }); + continue; + } + println!("==> Creating Lima disk {} ({})", disk.name, disk.size); + checked( + Command::new("limactl") + .args(["--tty=false", "disk", "create"]) + .arg(&disk.name) + .args(["--size", &disk.size]), + "create the Lima development cache disk", + )?; + attached.push(AttachedDisk { + request: disk.clone(), + format: true, + }); + } + Ok(attached) +} + +fn disk_list_contains(output: &[u8], name: &str) -> bool { + let needle = format!(r#""name":"{name}""#); + String::from_utf8_lossy(output) + .lines() + .any(|line| line.contains(&needle)) +} + +fn validate_disk_name(name: &str) -> Result<(), String> { + if !name.is_empty() + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + Ok(()) + } else { + Err(format!("invalid Lima disk name: {name}")) + } +} + +fn configure_disks(process: &mut Command, disks: &[AttachedDisk]) -> Result<(), String> { + if disks.is_empty() { + return Ok(()); + } + + let mut entries = Vec::with_capacity(disks.len()); + for disk in disks { + validate_disk_name(&disk.request.name)?; + entries.push(format!( + r#"{{"name":"{}","format":{},"fsType":"{}"}}"#, + disk.request.name, disk.format, disk.request.filesystem + )); + } + process + .arg("--set") + .arg(format!(".additionalDisks = [{}]", entries.join(","))); + Ok(()) +} + +fn disable_disk_format(instance: &str, disks: &[AttachedDisk]) -> Result<(), String> { + if !disks.iter().any(|disk| disk.format) { + return Ok(()); + } + checked( + Command::new("limactl").args([ + "--tty=false", + "edit", + instance, + "--set", + ".additionalDisks[].format = false", + ]), + "disable repeated Lima development cache disk formatting", + ) +} + +fn driver_available(driver: &str) -> Result { + let output = Command::new("limactl") + .args(["start", "--list-drivers"]) + .output() + .map_err(|error| format!("failed to list Lima VM drivers: {error}"))?; + if !output.status.success() { + return Err("failed to list Lima VM drivers".to_owned()); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .any(|available| available.trim() == driver)) +} + +fn instance_status(instance: &str) -> Result, String> { + let output = Command::new("limactl") + .args(["list", "--format", "{{.Name}}\t{{.Status}}"]) + .output() + .map_err(|error| format!("failed to inspect Lima instance {instance}: {error}"))?; + if !output.status.success() { + return Err(format!("failed to inspect Lima instance {instance}")); + } + + Ok(parse_instance_status(&output.stdout, instance)) +} + +fn parse_instance_status(output: &[u8], instance: &str) -> Option { + String::from_utf8_lossy(output).lines().find_map(|line| { + let (name, status) = line.split_once('\t')?; + (name == instance).then(|| status.to_owned()) + }) +} + +fn snapshot_exists(instance: &str, tag: &str) -> Result { + let output = Command::new("limactl") + .args(["snapshot", "list", instance, "--quiet"]) + .output() + .map_err(|error| format!("failed to list Lima snapshots for {instance}: {error}"))?; + if !output.status.success() { + return Err(format!("failed to list Lima snapshots for {instance}")); + } + + Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .any(|snapshot| snapshot.trim() == tag)) +} + +fn stop_instance(instance: &str) -> Result<(), String> { + if instance_status(instance)?.as_deref() != Some("Running") { + return Ok(()); + } + + checked( + Command::new("limactl").args(["--tty=false", "stop", instance]), + "stop the Lima test instance", + ) +} + +fn delete_instance(instance: &str, description: &str) -> Result<(), String> { + checked( + Command::new("limactl").args(["--tty=false", "delete", "--force", instance]), + description, + ) +} + +fn guest_shell_command(instance: &str, forward_env: &[&str]) -> Command { + let mut command = Command::new("limactl"); + command.args(["--tty=false", "shell"]); + if !forward_env.is_empty() { + command + .arg("--preserve-env") + .env("LIMA_SHELLENV_BLOCK", "*") + .env("LIMA_SHELLENV_ALLOW", forward_env.join(",")); + } + command.args([instance, "bash", "-s"]); + command +} + +fn run_guest_script( + instance: &str, + script: &str, + description: &str, + forward_env: &[&str], +) -> Result { + let mut child = guest_shell_command(instance, forward_env) + .stdin(Stdio::piped()) + .spawn() + .map_err(|error| format!("failed to start the Lima guest {description}: {error}"))?; + + child + .stdin + .take() + .ok_or_else(|| "failed to open stdin for the Lima guest command".to_owned())? + .write_all(script.as_bytes()) + .map_err(|error| format!("failed to send the {description} to Lima: {error}"))?; + + child + .wait() + .map_err(|error| format!("failed to wait for the Lima guest {description}: {error}")) +} + +fn checked(command: &mut Command, description: &str) -> Result<(), String> { + let status = command + .status() + .map_err(|error| format!("failed to execute {description}: {error}"))?; + if status.success() { + Ok(()) + } else { + Err(format!( + "{description} failed with exit code {}", + display_exit_code(status) + )) + } +} + +fn display_exit_code(status: ExitStatus) -> String { + status + .code() + .map_or_else(|| "signal".to_owned(), |code| code.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::machine::ReusePolicy; + + fn podman_request(os: MachineOs) -> MachineRequest { + MachineRequest { + os, + arch: Arch::Arm64, + purpose: "e2e", + profile: "podman", + forward_env: Vec::new(), + keep_on_failure: false, + reuse: ReusePolicy::ReusePrepared { + rebuild: false, + preparation_key: 0, + }, + host_mounts: Vec::new(), + persistent_disks: Vec::new(), + } + } + + #[test] + fn owns_the_target_to_template_mapping() { + assert_eq!( + lima_template(MachineOs::CentosStream10), + "template:centos-stream-10" + ); + assert_eq!( + lima_template(MachineOs::Ubuntu24_04), + "template:ubuntu-24.04" + ); + assert_eq!( + lima_template(MachineOs::Ubuntu26_04), + "template:ubuntu-26.04" + ); + } + + #[test] + fn forwards_only_requested_host_environment() { + let command = guest_shell_command( + "openshell-e2e-ubuntu-24.04-12345678", + &["MISE_GITHUB_TOKEN"], + ); + let args = command + .get_args() + .map(|argument| argument.to_string_lossy()) + .collect::>(); + + assert_eq!( + args, + [ + "--tty=false", + "shell", + "--preserve-env", + "openshell-e2e-ubuntu-24.04-12345678", + "bash", + "-s", + ] + ); + assert!(command.get_envs().any(|(name, value)| { + name == "LIMA_SHELLENV_BLOCK" && value == Some(std::ffi::OsStr::new("*")) + })); + assert!(command.get_envs().any(|(name, value)| { + name == "LIMA_SHELLENV_ALLOW" + && value == Some(std::ffi::OsStr::new("MISE_GITHUB_TOKEN")) + })); + } + + #[test] + fn cleans_up_failed_provisioning_unless_the_vm_was_requested() { + let cleaned = std::cell::Cell::new(false); + let result: Result<(), String> = finish_preparation( + Err("setup failed".to_owned()), + "test-vm", + false, + "Cleaning test VM", + || { + cleaned.set(true); + Ok(()) + }, + ); + assert_eq!(result.expect_err("setup should fail"), "setup failed"); + assert!(cleaned.get()); + + cleaned.set(false); + let result: Result<(), String> = finish_preparation( + Err("setup failed".to_owned()), + "test-vm", + true, + "Cleaning test VM", + || { + cleaned.set(true); + Ok(()) + }, + ); + assert_eq!(result.expect_err("setup should fail"), "setup failed"); + assert!(!cleaned.get()); + } + + #[test] + fn reports_a_provisioning_cleanup_failure() { + let result: Result<(), String> = finish_preparation( + Err("setup failed".to_owned()), + "test-vm", + false, + "Cleaning test VM", + || Err("delete failed".to_owned()), + ); + + assert_eq!( + result.expect_err("setup and cleanup should fail"), + "setup failed; cleanup also failed: delete failed" + ); + } + + #[test] + fn names_the_vm_for_its_environment() { + let request = podman_request(MachineOs::Ubuntu24_04); + let reusable = instance_name(&request, Backend::Qemu, false); + let suffix = reusable + .strip_prefix("openshell-e2e-ubuntu-24.04-") + .expect("name should expose its purpose and machine OS"); + assert_eq!(suffix.len(), 8); + assert!(suffix.bytes().all(|byte| byte.is_ascii_hexdigit())); + assert_ne!(reusable, instance_name(&request, Backend::Qemu, true)); + } + + #[test] + fn leaves_room_for_lima_socket_paths() { + let request = podman_request(MachineOs::Ubuntu24_04); + + // Lima appends a PID and temporary socket suffix beneath ~/.lima. Keep + // the stable portion bounded so common macOS home paths remain below + // UNIX_PATH_MAX even with a ten-digit PID. + assert!(instance_name(&request, Backend::Default, false).len() <= 40); + let mut mounted = request.clone(); + mounted.host_mounts = vec![HostMount::read_only("/tmp/OpenShell")]; + assert!(instance_name(&mounted, Backend::Default, false).len() <= 40); + let mut smoke = request; + smoke.purpose = "smoke"; + assert!(instance_name(&smoke, Backend::Default, false).len() <= 40); + assert!( + instance_name(&smoke, Backend::Default, false) + .starts_with("openshell-smoke-ubuntu-24.04-") + ); + } + + #[test] + fn keys_mounted_instances_to_the_source_path() { + let mut first_request = podman_request(MachineOs::Ubuntu24_04); + first_request.host_mounts = vec![HostMount::read_only("/work/first/OpenShell")]; + let mut second_request = first_request.clone(); + second_request.host_mounts = vec![HostMount::read_only("/work/second/OpenShell")]; + let mut cached_request = first_request.clone(); + cached_request.host_mounts.push(HostMount { + path: "/cache/sccache".into(), + writable: true, + }); + + let first = instance_name(&first_request, Backend::Qemu, false); + let second = instance_name(&second_request, Backend::Qemu, false); + let first_with_cache = instance_name(&cached_request, Backend::Qemu, false); + + assert_ne!(first, second); + assert_ne!(first, first_with_cache); + assert!(first.starts_with("openshell-e2e-ubuntu-24.04-")); + } + + #[test] + fn enables_the_guest_agent_only_for_mounted_instances() { + let mut mounted = Command::new("limactl"); + configure_mount_mode( + &mut mounted, + &[ + HostMount::read_only("/work/OpenShell"), + HostMount { + path: "/cache/sccache".into(), + writable: true, + }, + ], + ); + let mounted_args = mounted.get_args().collect::>(); + assert!(mounted_args.contains(&std::ffi::OsStr::new("--mount-only"))); + assert!(mounted_args.contains(&std::ffi::OsStr::new("--containerd"))); + assert!(mounted_args.contains(&std::ffi::OsStr::new("/work/OpenShell"))); + assert!(mounted_args.contains(&std::ffi::OsStr::new("/cache/sccache:w"))); + assert!(!mounted_args.contains(&std::ffi::OsStr::new("--plain"))); + + let mut unmounted = Command::new("limactl"); + configure_mount_mode(&mut unmounted, &[]); + let unmounted_args = unmounted.get_args().collect::>(); + assert!(unmounted_args.contains(&std::ffi::OsStr::new("--plain"))); + assert!(!unmounted_args.contains(&std::ffi::OsStr::new("--mount-only"))); + } + + #[test] + fn keys_snapshots_to_the_complete_mount_configuration() { + let request = podman_request(MachineOs::Ubuntu24_04); + let source = HostMount::read_only("/work/OpenShell"); + let disk = PersistentDisk::ext4("cache-a64-12345678", "50GiB"); + let base = snapshot_tag(&request, "setup"); + assert!(base.starts_with("base-v5-")); + + let mut mounted = request.clone(); + mounted.host_mounts = vec![source]; + assert_ne!(snapshot_tag(&mounted, "setup"), base); + + let mut with_disk = mounted.clone(); + with_disk.persistent_disks = vec![disk]; + assert_ne!( + snapshot_tag(&with_disk, "setup"), + snapshot_tag(&mounted, "setup") + ); + + let mut larger_disk = mounted.clone(); + larger_disk.persistent_disks = vec![PersistentDisk::ext4("cache-a64-12345678", "60GiB")]; + assert_ne!( + snapshot_tag(&with_disk, "setup"), + snapshot_tag(&larger_disk, "setup") + ); + assert_ne!(base, snapshot_tag(&request, "changed setup")); + + let mut changed_key = request.clone(); + changed_key.reuse = ReusePolicy::ReusePrepared { + rebuild: false, + preparation_key: 1, + }; + assert_ne!(base, snapshot_tag(&changed_key, "setup")); + + let mut changed_profile = request.clone(); + changed_profile.profile = "another-suite"; + assert_ne!(base, snapshot_tag(&changed_profile, "setup")); + } + + #[test] + fn configures_and_detects_persistent_disks() { + let disk = PersistentDisk::ext4("cache-a64-12345678", "50GiB"); + let attached = AttachedDisk { + request: disk.clone(), + format: false, + }; + let mut command = Command::new("limactl"); + configure_disks(&mut command, std::slice::from_ref(&attached)) + .expect("disk should configure"); + let arguments = command + .get_args() + .map(|argument| argument.to_string_lossy()) + .collect::>(); + + assert_eq!(arguments[0], "--set"); + assert!(arguments[1].contains(r#""name":"cache-a64-12345678""#)); + assert!(arguments[1].contains(r#""fsType":"ext4""#)); + assert!(arguments[1].contains(r#""format":false"#)); + assert!(disk_list_contains( + br#"{"name":"cache-a64-12345678","size":53687091200}"#, + &disk.name + )); + assert!(!disk_list_contains( + br#"{"name":"cache-a64-87654321","size":53687091200}"#, + &disk.name + )); + + let new_disk = AttachedDisk { + request: disk.clone(), + format: true, + }; + let mut new_command = Command::new("limactl"); + configure_disks(&mut new_command, std::slice::from_ref(&new_disk)) + .expect("new disk should configure"); + assert!( + new_command + .get_args() + .any(|argument| argument.to_string_lossy().contains(r#""format":true"#)) + ); + let mut request = podman_request(MachineOs::Ubuntu24_04); + request.persistent_disks = vec![disk.clone()]; + let mut new_request = request.clone(); + new_request.persistent_disks = vec![disk]; + assert_eq!( + snapshot_tag(&request, "setup"), + snapshot_tag(&new_request, "setup") + ); + } + + #[test] + fn parses_matching_instance_status() { + let output = b"other\tStopped\nopenshell-e2e-ubuntu-24.04-12345678\tRunning\n"; + assert_eq!( + parse_instance_status(output, "openshell-e2e-ubuntu-24.04-12345678"), + Some("Running".to_owned()) + ); + assert_eq!(parse_instance_status(output, "missing"), None); + } +} diff --git a/crates/xtask/src/machine.rs b/crates/xtask/src/machine.rs new file mode 100644 index 0000000000..0c85b29d77 --- /dev/null +++ b/crates/xtask/src/machine.rs @@ -0,0 +1,348 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; +use std::process::ExitStatus; + +use crate::platform::{Arch, MachineOs}; + +pub const DEFAULT_MACHINE_OS: MachineOs = MachineOs::Ubuntu26_04; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Provider { + #[default] + Lima, +} + +impl Provider { + pub fn parse(value: &OsStr) -> Result { + match value.to_str() { + Some("lima") => Ok(Self::Lima), + Some(value) => Err(format!( + "unsupported machine provider: {value} (expected lima)" + )), + None => Err("--provider must be valid UTF-8".to_owned()), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReusePolicy { + Ephemeral, + ReusePrepared { rebuild: bool, preparation_key: u32 }, +} + +impl ReusePolicy { + pub const fn enabled(self) -> bool { + matches!(self, Self::ReusePrepared { .. }) + } + + pub const fn rebuild(self) -> bool { + match self { + Self::Ephemeral => false, + Self::ReusePrepared { rebuild, .. } => rebuild, + } + } + + pub const fn preparation_key(self) -> u32 { + match self { + Self::Ephemeral => 0, + Self::ReusePrepared { + preparation_key, .. + } => preparation_key, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MachineOptions { + pub arch: Option, + pub keep: bool, + pub provider: Provider, + pub rebuild: bool, + pub snapshot: bool, + pub os: MachineOs, +} + +impl MachineOptions { + pub const fn reuse_policy(&self, preparation_key: u32) -> ReusePolicy { + if self.snapshot { + ReusePolicy::ReusePrepared { + rebuild: self.rebuild, + preparation_key, + } + } else { + ReusePolicy::Ephemeral + } + } +} + +#[derive(Default)] +pub struct MachineOptionsBuilder { + arch: Option, + keep: bool, + provider: Option, + rebuild: bool, + snapshot: bool, +} + +impl MachineOptionsBuilder { + pub fn parse_argument( + &mut self, + argument: &str, + args: &mut impl Iterator, + ) -> Result { + match argument { + "--arch" => { + let value = args + .next() + .ok_or_else(|| "--arch requires a value".to_owned())?; + if self + .arch + .replace(crate::platform::parse_arch(&value)?) + .is_some() + { + return Err("--arch may only be specified once".to_owned()); + } + } + "--provider" => { + let value = args + .next() + .ok_or_else(|| "--provider requires a value".to_owned())?; + if self.provider.replace(Provider::parse(&value)?).is_some() { + return Err("--provider may only be specified once".to_owned()); + } + } + "--keep-machine" => self.keep = true, + "--rebuild-machine" => self.rebuild = true, + "--snapshot" => self.snapshot = true, + _ => return Ok(false), + } + Ok(true) + } + + pub fn finish(self, os: Option) -> Result, String> { + let os = match os { + Some(os) => os, + None if self.provider.is_some() => DEFAULT_MACHINE_OS, + None => { + let option = if self.arch.is_some() { + Some("--arch") + } else if self.keep { + Some("--keep-machine") + } else if self.rebuild { + Some("--rebuild-machine") + } else if self.snapshot { + Some("--snapshot") + } else { + None + }; + return option.map_or(Ok(None), |option| { + Err(format!("{option} requires --os or --provider")) + }); + } + }; + + if self.rebuild && !self.snapshot { + return Err("--rebuild-machine requires --snapshot".to_owned()); + } + + Ok(Some(MachineOptions { + arch: self.arch, + keep: self.keep, + provider: self.provider.unwrap_or_default(), + rebuild: self.rebuild, + snapshot: self.snapshot, + os, + })) + } +} + +pub(crate) fn path_hash(path: &Path) -> u32 { + // FNV-1a keeps mounted-worktree names stable across processes and Rust + // versions without pulling a hashing dependency into xtask. + path.as_os_str() + .as_encoded_bytes() + .iter() + .fold(0x811c9dc5_u32, |hash, byte| { + (hash ^ u32::from(*byte)).wrapping_mul(0x01000193) + }) +} + +pub(crate) fn stable_hash(parts: &[&[u8]]) -> u32 { + parts.iter().fold(0x811c9dc5_u32, |hash, part| { + let hash = part.iter().fold(hash, |hash, byte| { + (hash ^ u32::from(*byte)).wrapping_mul(0x01000193) + }); + (hash ^ 0xff).wrapping_mul(0x01000193) + }) +} + +#[derive(Debug, Clone)] +pub struct MachineRequest { + pub os: MachineOs, + pub arch: Arch, + pub purpose: &'static str, + pub profile: &'static str, + pub forward_env: Vec<&'static str>, + pub keep_on_failure: bool, + pub reuse: ReusePolicy, + pub host_mounts: Vec, + pub persistent_disks: Vec, +} + +pub(crate) fn hash_request_resources(hash: u32, request: &MachineRequest) -> u32 { + let hash = request.host_mounts.iter().fold(hash, |hash, mount| { + let hash = mount + .path + .as_os_str() + .as_encoded_bytes() + .iter() + .fold(hash, |hash, byte| { + (hash ^ u32::from(*byte)).wrapping_mul(0x01000193) + }); + [u8::from(mount.writable), 0xff] + .iter() + .fold(hash, |hash, byte| { + (hash ^ u32::from(*byte)).wrapping_mul(0x01000193) + }) + }); + request.persistent_disks.iter().fold(hash, |hash, disk| { + let hash = + [&disk.name, &disk.size, disk.filesystem] + .into_iter() + .fold(hash, |hash, value| { + let hash = value.as_bytes().iter().fold(hash, |hash, byte| { + (hash ^ u32::from(*byte)).wrapping_mul(0x01000193) + }); + (hash ^ 0xfe).wrapping_mul(0x01000193) + }); + (hash ^ 0xff).wrapping_mul(0x01000193) + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostMount { + pub path: PathBuf, + pub writable: bool, +} + +impl HostMount { + pub fn read_only(path: impl Into) -> Self { + Self { + path: path.into(), + writable: false, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PersistentDisk { + pub name: String, + pub size: String, + pub filesystem: &'static str, +} + +impl PersistentDisk { + pub fn ext4(name: impl Into, size: impl Into) -> Self { + Self { + name: name.into(), + size: size.into(), + filesystem: "ext4", + } + } +} + +pub trait MachineProvider { + fn persistent_disk_mount_point(&self, disk: &PersistentDisk) -> String; + + fn acquire( + &self, + request: MachineRequest, + setup_script: &str, + ) -> Result, String>; +} + +pub trait Machine { + fn name(&self) -> &str; + + fn copy_file(&self, source: &Path, destination: &str) -> Result<(), String>; + + fn run_script(&self, script: &str, description: &str) -> Result; + + fn release(&self) -> Result<(), String>; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_and_defaults_the_machine_provider() { + assert_eq!(Provider::default(), Provider::Lima); + assert_eq!(Provider::parse(OsStr::new("lima")), Ok(Provider::Lima)); + assert!(Provider::parse(OsStr::new("cloud")).is_err()); + } + + #[test] + fn builds_shared_machine_options() { + let options = MachineOptionsBuilder::default() + .finish(Some(MachineOs::Ubuntu24_04)) + .expect("default machine options should be valid") + .expect("an OS should produce machine options"); + + assert_eq!(options.arch, None); + assert_eq!(options.provider, Provider::Lima); + assert_eq!(options.os, MachineOs::Ubuntu24_04); + assert_eq!(options.reuse_policy(42), ReusePolicy::Ephemeral); + } + + #[test] + fn a_provider_uses_the_default_machine_target() { + let builder = MachineOptionsBuilder { + provider: Some(Provider::Lima), + ..Default::default() + }; + + let options = builder + .finish(None) + .expect("provider-only machine options should be valid") + .expect("a provider should request a machine"); + + assert_eq!(options.os, MachineOs::Ubuntu26_04); + } + + #[test] + fn rebuilding_a_machine_requires_snapshots() { + let builder = MachineOptionsBuilder { + rebuild: true, + ..Default::default() + }; + + let error = builder + .finish(Some(MachineOs::Ubuntu24_04)) + .expect_err("rebuilding without snapshots should fail"); + assert!(error.contains("--rebuild-machine requires --snapshot")); + } + + #[test] + fn exposes_semantic_machine_reuse() { + assert!(!ReusePolicy::Ephemeral.enabled()); + assert!(!ReusePolicy::Ephemeral.rebuild()); + assert_eq!(ReusePolicy::Ephemeral.preparation_key(), 0); + + let reuse = ReusePolicy::ReusePrepared { + rebuild: true, + preparation_key: 42, + }; + assert!(reuse.enabled()); + assert!(reuse.rebuild()); + assert_eq!(reuse.preparation_key(), 42); + } + + #[test] + fn stable_hash_separates_inputs() { + assert_ne!(stable_hash(&[b"ab", b"c"]), stable_hash(&[b"a", b"bc"])); + } +} diff --git a/crates/xtask/src/main.rs b/crates/xtask/src/main.rs new file mode 100644 index 0000000000..27ed1385d5 --- /dev/null +++ b/crates/xtask/src/main.rs @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::env; +use std::process::ExitCode; + +mod e2e; +mod e2e_machine; +mod lima; +mod machine; +mod platform; +mod provider; +mod release_smoke_test; +pub mod tasks; + +fn main() -> ExitCode { + let mut args = env::args_os().skip(1); + let result = match args.next() { + None => { + tasks::print_help(); + return ExitCode::SUCCESS; + } + Some(task) => match task.to_str() { + Some("help" | "-h" | "--help") => { + tasks::print_help(); + return ExitCode::SUCCESS; + } + Some("e2e") => tasks::e2e(args), + Some("release-smoke-test") => tasks::release_smoke_test(args), + Some(invalid) => Err(format!("invalid task name: {invalid}")), + None => Err("task name must be valid UTF-8".to_owned()), + }, + }; + + match result { + Ok(exit_code) => exit_code, + Err(message) => { + eprintln!("error: {message}"); + tasks::print_help(); + ExitCode::FAILURE + } + } +} diff --git a/crates/xtask/src/platform.rs b/crates/xtask/src/platform.rs new file mode 100644 index 0000000000..51b9a15d6f --- /dev/null +++ b/crates/xtask/src/platform.rs @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::OsStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Arch { + Amd64, + Arm64, +} + +pub fn parse_arch(value: &OsStr) -> Result { + match value.to_str() { + Some("amd64" | "x86_64") => Ok(Arch::Amd64), + Some("arm64" | "aarch64") => Ok(Arch::Arm64), + Some(value) => Err(format!("unsupported architecture: {value}")), + None => Err("--arch must be valid UTF-8".to_owned()), + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OsFamily { + CentosStream, + Ubuntu, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MachineOs { + CentosStream10, + Ubuntu24_04, + Ubuntu26_04, +} + +impl MachineOs { + pub const fn id(self) -> &'static str { + match self { + Self::CentosStream10 => "centos-stream-10", + Self::Ubuntu24_04 => "ubuntu-24.04", + Self::Ubuntu26_04 => "ubuntu-26.04", + } + } + + pub const fn short_name(self) -> &'static str { + match self { + Self::CentosStream10 => "centos10", + Self::Ubuntu24_04 => "ubuntu2404", + Self::Ubuntu26_04 => "ubuntu2604", + } + } + + pub const fn family(self) -> OsFamily { + match self { + Self::CentosStream10 => OsFamily::CentosStream, + Self::Ubuntu24_04 | Self::Ubuntu26_04 => OsFamily::Ubuntu, + } + } +} + +pub fn parse_machine_os(value: &OsStr, option: &str) -> Result { + match value.to_str() { + Some("centos-stream-10") => Ok(MachineOs::CentosStream10), + Some("ubuntu-24.04") => Ok(MachineOs::Ubuntu24_04), + Some("ubuntu-26.04") => Ok(MachineOs::Ubuntu26_04), + Some(value) => Err(format!( + "unsupported machine OS: {value} (expected centos-stream-10, ubuntu-24.04, or ubuntu-26.04)" + )), + None => Err(format!("{option} must be valid UTF-8")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_supported_target_operating_systems() { + assert_eq!( + parse_machine_os(OsStr::new("centos-stream-10"), "--os"), + Ok(MachineOs::CentosStream10) + ); + assert_eq!( + parse_machine_os(OsStr::new("ubuntu-24.04"), "--os"), + Ok(MachineOs::Ubuntu24_04) + ); + assert_eq!( + parse_machine_os(OsStr::new("ubuntu-26.04"), "--os"), + Ok(MachineOs::Ubuntu26_04) + ); + } + + #[test] + fn rejects_unsupported_target_operating_systems() { + let error = parse_machine_os(OsStr::new("debian-13"), "--os") + .expect_err("unsupported machine OS should fail"); + assert!(error.contains("unsupported machine OS: debian-13")); + assert!(error.contains("expected centos-stream-10, ubuntu-24.04, or ubuntu-26.04")); + } +} diff --git a/crates/xtask/src/provider.rs b/crates/xtask/src/provider.rs new file mode 100644 index 0000000000..21c57bfdce --- /dev/null +++ b/crates/xtask/src/provider.rs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::lima::LimaProvider; +use crate::machine::{Machine, MachineProvider, MachineRequest, PersistentDisk, Provider}; + +impl MachineProvider for Provider { + fn persistent_disk_mount_point(&self, disk: &PersistentDisk) -> String { + match self { + Self::Lima => LimaProvider.persistent_disk_mount_point(disk), + } + } + + fn acquire( + &self, + request: MachineRequest, + setup_script: &str, + ) -> Result, String> { + match self { + Self::Lima => LimaProvider.acquire(request, setup_script), + } + } +} diff --git a/crates/xtask/src/release_smoke_test.rs b/crates/xtask/src/release_smoke_test.rs new file mode 100644 index 0000000000..408424c149 --- /dev/null +++ b/crates/xtask/src/release_smoke_test.rs @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::env; +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; +use std::process::{ExitCode, ExitStatus}; + +use crate::machine::{ + DEFAULT_MACHINE_OS, MachineOptions, MachineOptionsBuilder, MachineProvider, MachineRequest, +}; +use crate::platform::{Arch, MachineOs, OsFamily, parse_machine_os}; +use crate::tasks::{TaskResult, exit_code, print_help_if_requested}; + +const RELEASE_SMOKE_UBUNTU_PODMAN_ROOTLESS_SCRIPT: &str = + include_str!("../scripts/release-smoke/ubuntu-podman-rootless.sh"); +const UBUNTU_OS_SETUP: &str = include_str!("../scripts/machine/os/ubuntu.sh"); +const UBUNTU_PODMAN_SETUP: &str = include_str!("../scripts/machine/suites/podman/ubuntu.sh"); +const COMMON_PODMAN_SETUP: &str = include_str!("../scripts/machine/suites/podman/common.sh"); +const GUEST_RELEASE_ARTIFACT_PATH: &str = "/tmp/openshell-release.deb"; +const HELP: &str = "Test a Debian release artifact on a target machine. + +Usage: + cargo xtask release-smoke-test --deb [--provider ] [--arch ] [--os ] [--snapshot] [--rebuild-machine] [--keep-machine] + +Defaults: + --provider lima + --os ubuntu-26.04"; + +pub fn run(args: impl Iterator) -> TaskResult { + let mut args = args.peekable(); + if print_help_if_requested(&mut args, HELP) { + return Ok(ExitCode::SUCCESS); + } + + let command = ReleaseSmokeTestCommand::parse(args)?; + release_smoke_test(&command.machine.provider, &command).map(exit_code) +} + +struct ReleaseSmokeTestCommand { + deb: PathBuf, + machine: MachineOptions, +} + +impl ReleaseSmokeTestCommand { + fn parse(mut args: impl Iterator) -> Result { + let mut deb = None; + let mut machine = MachineOptionsBuilder::default(); + let mut os = None; + + while let Some(argument) = args.next() { + match argument.to_str() { + Some("--deb") => { + let value = args + .next() + .ok_or_else(|| "--deb requires a path".to_owned())?; + if deb.replace(PathBuf::from(value)).is_some() { + return Err("--deb may only be specified once".to_owned()); + } + } + Some("--os") => { + let value = args + .next() + .ok_or_else(|| "--os requires a value".to_owned())?; + if os.replace(parse_machine_os(&value, "--os")?).is_some() { + return Err("--os may only be specified once".to_owned()); + } + } + Some(value) if machine.parse_argument(value, &mut args)? => {} + Some(value) => return Err(format!("unknown release-smoke-test option: {value}")), + None => return Err("release-smoke-test options must be valid UTF-8".to_owned()), + } + } + + let os = os.unwrap_or(DEFAULT_MACHINE_OS); + if !matches!(os.family(), OsFamily::Ubuntu) { + return Err(format!( + "release-smoke-test does not support --os {} (expected ubuntu-24.04 or ubuntu-26.04)", + os.id() + )); + } + + Ok(Self { + deb: deb.ok_or_else(|| "release-smoke-test requires --deb ".to_owned())?, + machine: machine + .finish(Some(os))? + .expect("release smoke tests always request a target machine"), + }) + } +} + +fn release_smoke_test( + provider: &P, + command: &ReleaseSmokeTestCommand, +) -> Result { + let deb = command.deb.canonicalize().map_err(|error| { + format!( + "cannot read Debian artifact {}: {error}", + command.deb.display() + ) + })?; + if !deb.is_file() { + return Err(format!("Debian artifact is not a file: {}", deb.display())); + } + + let options = &command.machine; + let arch = options.arch.unwrap_or_else(|| infer_deb_arch(&deb)); + let setup_script = release_setup_script(options.os)?; + let test_script = release_smoke_guest_script(options.os)?; + let machine = provider.acquire( + MachineRequest { + os: options.os, + arch, + purpose: "smoke", + profile: "podman-rl", + forward_env: Vec::new(), + keep_on_failure: options.keep, + reuse: options.reuse_policy(0), + host_mounts: Vec::new(), + persistent_disks: Vec::new(), + }, + &setup_script, + )?; + + println!("==> Testing {} with {}", deb.display(), machine.name()); + + let test_result = (|| { + machine.copy_file(&deb, GUEST_RELEASE_ARTIFACT_PATH)?; + machine.run_script(&test_script, "release smoke test") + })(); + + if options.keep { + eprintln!("Machine kept for inspection: {}", machine.name()); + return test_result; + } + + let release_result = machine.release(); + match (test_result, release_result) { + (Ok(status), Ok(())) => Ok(status), + (Err(error), Ok(())) => Err(error), + (Err(error), Err(release_error)) => { + Err(format!("{error}; release also failed: {release_error}")) + } + (Ok(status), Err(error)) if status.success() => Err(error), + (Ok(status), Err(error)) => { + eprintln!("warning: {error}"); + Ok(status) + } + } +} + +fn release_smoke_guest_script(os: MachineOs) -> Result { + match os.family() { + OsFamily::CentosStream => Err(format!( + "release-smoke-test does not support --os {}", + os.id() + )), + OsFamily::Ubuntu => Ok(format!( + "export OPENSHELL_RELEASE_ARTIFACT={GUEST_RELEASE_ARTIFACT_PATH}\n\ + {RELEASE_SMOKE_UBUNTU_PODMAN_ROOTLESS_SCRIPT}" + )), + } +} + +fn release_setup_script(os: MachineOs) -> Result { + match os.family() { + OsFamily::CentosStream => Err(format!( + "release-smoke-test does not support --os {}", + os.id() + )), + OsFamily::Ubuntu => { + Ok([UBUNTU_OS_SETUP, UBUNTU_PODMAN_SETUP, COMMON_PODMAN_SETUP].join("\n")) + } + } +} + +fn infer_deb_arch(path: &Path) -> Arch { + let filename = path.file_name().and_then(OsStr::to_str).unwrap_or_default(); + if filename.ends_with("_arm64.deb") || filename.ends_with("-arm64.deb") { + return Arch::Arm64; + } + if filename.ends_with("_amd64.deb") || filename.ends_with("-amd64.deb") { + return Arch::Amd64; + } + + match env::consts::ARCH { + "aarch64" => Arch::Arm64, + _ => Arch::Amd64, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::machine::Provider; + + #[test] + fn parses_release_smoke_test_options() { + let command = ReleaseSmokeTestCommand::parse( + [ + "--deb", + "artifacts/openshell_1.2.3_arm64.deb", + "--provider", + "lima", + "--arch", + "arm64", + "--keep-machine", + "--rebuild-machine", + "--snapshot", + "--os", + "ubuntu-24.04", + ] + .into_iter() + .map(OsString::from), + ) + .expect("command should parse"); + + assert_eq!( + command.deb, + PathBuf::from("artifacts/openshell_1.2.3_arm64.deb") + ); + assert_eq!(command.machine.arch, Some(Arch::Arm64)); + assert!(command.machine.keep); + assert_eq!(command.machine.provider, Provider::Lima); + assert!(command.machine.rebuild); + assert!(command.machine.snapshot); + assert_eq!(command.machine.os, MachineOs::Ubuntu24_04); + } + + #[test] + fn release_smoke_test_requires_deb() { + let error = ReleaseSmokeTestCommand::parse(std::iter::empty()) + .err() + .expect("missing --deb should fail"); + + assert!(error.contains("requires --deb")); + } + + #[test] + fn release_smoke_test_rejects_non_ubuntu_targets() { + let error = ReleaseSmokeTestCommand::parse( + [ + "--deb", + "artifacts/openshell_1.2.3_arm64.deb", + "--os", + "centos-stream-10", + ] + .into_iter() + .map(OsString::from), + ) + .err() + .expect("Debian release tests should reject CentOS Stream"); + + assert!(error.contains("release-smoke-test does not support --os centos-stream-10")); + } + + #[test] + fn release_smoke_test_defaults_to_ubuntu_26_04() { + let command = ReleaseSmokeTestCommand::parse( + ["--deb", "artifacts/openshell_1.2.3_arm64.deb"] + .into_iter() + .map(OsString::from), + ) + .expect("command should parse"); + + assert_eq!(command.machine.provider, Provider::Lima); + assert_eq!(command.machine.os, MachineOs::Ubuntu26_04); + } + + #[test] + fn infers_debian_architecture_from_artifact_name() { + assert_eq!( + infer_deb_arch(Path::new("openshell_1.2.3_arm64.deb")), + Arch::Arm64 + ); + assert_eq!( + infer_deb_arch(Path::new("openshell_1.2.3_amd64.deb")), + Arch::Amd64 + ); + } + + #[test] + fn selects_the_release_smoke_script_by_os_and_driver() { + let script = release_smoke_guest_script(MachineOs::Ubuntu24_04) + .expect("Ubuntu release smoke test should be supported"); + + assert!(script.contains("Creating a sandbox and verifying default-deny networking")); + assert!(script.contains("Installing Ubuntu release artifact")); + assert!(script.contains("export OPENSHELL_RELEASE_ARTIFACT=/tmp/openshell-release.deb")); + } +} diff --git a/crates/xtask/src/tasks/mod.rs b/crates/xtask/src/tasks/mod.rs new file mode 100644 index 0000000000..9a857e57af --- /dev/null +++ b/crates/xtask/src/tasks/mod.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::OsString; +use std::iter::Peekable; +use std::process::{ExitCode, ExitStatus}; + +pub use crate::e2e::run as e2e; +pub use crate::release_smoke_test::run as release_smoke_test; + +const HELP: &str = "OpenShell development tasks + +Usage: + cargo xtask [options] + +Commands: + e2e Run an e2e suite locally or on a target machine + release-smoke-test Test a Debian release artifact on a target machine + +Run `cargo xtask --help` for command-specific usage."; + +pub type TaskResult = Result; + +pub fn print_help() { + println!("{HELP}"); +} + +pub(crate) fn print_help_if_requested(args: &mut Peekable, help: &str) -> bool +where + I: Iterator, +{ + if args.peek().is_some_and(is_help_argument) { + println!("{help}"); + return true; + } + + false +} + +fn is_help_argument(argument: &OsString) -> bool { + argument == "-h" || argument == "--help" +} + +pub(crate) fn exit_code(status: ExitStatus) -> ExitCode { + match status.code() { + Some(code) => ExitCode::from(u8::try_from(code).unwrap_or(1)), + None => ExitCode::FAILURE, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recognizes_command_help_arguments() { + assert!(is_help_argument(&OsString::from("-h"))); + assert!(is_help_argument(&OsString::from("--help"))); + assert!(!is_help_argument(&OsString::from("help"))); + } +} diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index 51e56b97c8..cd32136ae0 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -126,7 +126,10 @@ async fn sandbox_mounts_enabled_driver_config_bind() { "type": "bind", "source": bind_source, "target": BIND_TARGET, - "read_only": false + "read_only": false, + // Relabel caller-owned bind data for container access on enforcing + // hosts. Shared labeling remains safe if another sandbox reuses it. + "selinux_label": "shared" }); let driver_config = driver_config_mount_json(&driver, &bind_mount); // Host bind mounts are explicitly unsafe: this test validates driver mount diff --git a/flake.lock b/flake.lock index 7b9881771a..80cdf70f8a 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1779560665, - "narHash": "sha256-tpyBcxPpcQb8ukyNF7DoCwfSY3VPsxHoYwj00Cayv5o=", + "lastModified": 1783776592, + "narHash": "sha256-UgCQzxeWI75XM8G+hPrPh+MKzEPjG3SpAj7dtqSbksA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "64c08a7ca051951c8eae34e3e3cb1e202fe36786", + "rev": "e7a3ca8092b61ff85b6a45bf863ea2b2d6a661b3", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 13c4857bc6..237e3b4c19 100644 --- a/flake.nix +++ b/flake.nix @@ -32,6 +32,24 @@ inherit system; overlays = [ (import rust-overlay) ]; }; + limaForDevShell = + (pkgs.lima.override { + withAdditionalGuestAgents = true; + }).overrideAttrs + (previous: { + # Backport https://github.com/NixOS/nixpkgs/commit/5ce128c4d99036a72c5c4c2044a954ebcd8e0801 + # until the fix reaches nixos-unstable. + nativeBuildInputs = + (previous.nativeBuildInputs or [ ]) + ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isDarwin [ + pkgs.llvmPackages.lld + ]; + env = + (previous.env or { }) + // pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isDarwin { + NIX_CFLAGS_LINK = "-fuse-ld=${pkgs.lib.getExe' pkgs.llvmPackages.lld "ld64.lld"}"; + }; + }); rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; treefmtEval = treefmt-nix.lib.evalModule pkgs { projectRootFile = "flake.nix"; @@ -42,6 +60,8 @@ devShells.default = pkgs.mkShell { packages = with pkgs; [ rustToolchain + # Required for running xtasks that use a lima provider + limaForDevShell # Required to find packages pkg-config # Required for bindgen generation.