runner-exec
Source: crates/lvcs/src/runner/mod.rs.
/// Materialize `frontier` for one job into `dir`, the job's fresh directory
/// `.lvcs/tmp/runs/<job id>/` ([`job_dir`], 0700, owned by the
/// daemon's uid): one run directory per check, `<dir>/<check id>/`
/// ([`run_dir`]), whose `work/` holds the frontier's files that check's
/// `paths` and `uses` match, with the bytes the shell read through
/// `ReadAt`, each file mode 0755 when its document's executable bit is set
/// and 0644 otherwise, as the mirror gives it (decision 85, row 307),
/// beside a fresh `home/`, `cargo/`, and `tmp/`. A check sees
/// only its own declared inputs, so a reused result can't be wrong
/// because of a hidden one. `derived` names each derived document of
/// `frontier` (`/meta/derived` true at its heads there, as the shell read
/// it) with the SHA-256 of the bytes it is materialized as, which is how
/// it enters a check's `InputsHash` (draft row A).
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Materialize {
/// The frontier, less its documents deleted at their heads.
pub(crate) frontier: Frontier,
/// The job's directory.
pub(crate) dir: PathBuf,
/// Each derived document's address, with its bytes' SHA-256.
pub(crate) derived: BTreeMap<String, [u8; 32]>,
}
/// What `lvcs sandbox` runs, read as one line of JSON from standard input.
///
/// The spec comes through an inherited pipe, never from a path, and the
/// pipe stays open after it: its write end is the step's life line, and
/// when every write end is closed (the daemon cancelled the step, decision
/// 249 revised, or the daemon is gone) the sandbox kills its command, or
/// never spawns it, and exits `CANCELLED`. `command`
/// (argv, no shell) in
/// the run directory `run`'s `work/` (or `cwd` under it), killed after `timeout_s` seconds (at
/// most `CHECK_TIMEOUT_SECS` for a check, `GATED_TIMEOUT_SECS` for a
/// gated run). `run` must be a directory under `.lvcs/tmp/runs/` once
/// symlinks are resolved, and the sandbox refuses any other; `gated` picks
/// its profile (decision 142, the triggers row). The sandbox refuses to
/// run when its own uid is 0 (a daemon running as root). The sandbox's filesystem holds only `run`'s `work/`
/// (writable) at `/work`, its `home/`, `cargo/`, and `tmp/` at `/home`,
/// `/cargo`, and `/tmp` (writable, `nosuid`, `nodev`), read-only, `nosuid` binds of `/usr`, `/bin`,
/// `/sbin`, `/lib`, and `/lib64` where present (a symlink among them, as
/// in a merged-`/usr` image, is recreated as the same symlink), a
/// read-only `/etc` (row 307) of generated `passwd` and `group` (`root`
/// and the sandbox's own user `lvcs`, home `/home`), `nsswitch.conf`, and
/// `hosts` (`localhost`), with read-only, `nosuid`, `nodev` binds of the
/// host's `resolv.conf`, `ssl/certs/`, `pki/tls/certs/`, `pki/ca-trust/`,
/// `ca-certificates/`,
/// `ca-certificates.conf`, `alternatives/`, and `ld.so.cache` where present
/// (a symlink bound as what it names) and nothing else of the host's, and
/// `/dev/null`, `/dev/zero`, and `/dev/urandom`, read-only, `nosuid` binds
/// of the host's character devices, which the sandbox refuses to start
/// without (decision 134): a write still reaches the device, and a change
/// to a node's mode, times, or ACL fails. The command's standard input is
/// `/dev/null`. The sandbox spawns `lvcs sandbox` again as its init stage,
/// the spec passed on with `init` set, which is PID 1 of the step's PID
/// namespace and execs the command, so the command stays PID 1 (row 307).
/// Its `/proc` is a fresh `proc` mount (read-only, `nosuid`, `nodev`,
/// `noexec`) of
/// that PID namespace, listing only the step's own processes, so
/// `/proc/self` works. Where the kernel refuses that mount (a container
/// whose `/proc` has masked paths), the step says `lvcs: sandbox: no
/// /proc/self: <reason>` and its `/proc` holds only read-only binds of the
/// host's `/proc/cpuinfo`, `/proc/meminfo`, and `/proc/stat` (decision
/// 248), so a tool can count CPUs and memory, and `/lvcs/bin` holds a
/// `/bin/sh` shim for `node` that execs it by its absolute path, so
/// `process.execPath` is absolute (the shim directory exists only when
/// the fixed `PATH` holds node). The environment is cleared, then
/// `PATH` (`path`'s entries, then `/lvcs/bin` when the step has no
/// `/proc/self` and the shim directory exists, then the
/// fixed `/usr/local/bin:/usr/bin:/bin`),
/// `HOME=/home`, `CARGO_HOME=/cargo`, `TMPDIR=/tmp`, and `LANG=C`, then
/// `env` and `secrets` in order (decisions 245 and 248), every name
/// non-empty and free of `=` and NUL, every value free of NUL, and their
/// bytes with `path`'s within `ACTION_ENV_BYTES`. Under `.lvcs/tmp/runs/` (a
/// change action's run) the sandbox refuses a spec with `network` set or
/// any `secrets` (fail closed). A gated run's may set `network`, which
/// skips only the network namespace, and `secrets`, its credentials
/// (decision 142): this pipe is the only way a credential reaches the
/// sandbox, so its `Debug` form names the variables and never their
/// values. Before it spawns the init stage the sandbox sets `RLIMIT_AS`,
/// `RLIMIT_NPROC`, `RLIMIT_FSIZE`, `RLIMIT_CPU`, `RLIMIT_NOFILE`, and a zero
/// `RLIMIT_CORE` (the `CHECK_*` constants) and joins a cgroup leaf of its
/// own with `memory.max`, `pids.max`, and `cpu.max` when the daemon has a
/// delegated cgroup. The init stage, after its `proc` mount, drops every
/// capability, sets `no_new_privs`, and installs a seccomp deny-list whose
/// calls fail with `EPERM`: namespaces, mounts, `ptrace`, the keyring, BPF,
/// `perf_event_open`, `userfaultfd`, modules, `kexec`, and the like (the
/// sandbox-hardening row, production readiness S3), then execs the
/// command. A spec the daemon writes never sets `init`, and the sandbox
/// refuses `init` anywhere but PID 1 of its own PID namespace.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct SandboxSpec {
/// The run directory.
pub(crate) run: PathBuf,
/// The command's argv, run with no shell.
pub(crate) command: Vec<String>,
/// Its wall clock, in seconds, within its profile's.
pub(crate) timeout_s: u64,
/// Whether it is a gated action's run (one with `after`): the host's
/// network, the secrets below, and `GATED_TIMEOUT_SECS`.
pub(crate) gated: bool,
/// The variables set after the fixed ones, by name.
pub(crate) env: Vec<(String, String)>,
/// The variables a secret hole filled, set after `env` (a gated run
/// only).
pub(crate) secrets: Vec<(String, String)>,
/// The directories before the fixed `PATH`'s.
pub(crate) path: Vec<String>,
/// The working directory, relative to `/work`.
pub(crate) cwd: Option<String>,
/// Whether this is the sandbox's own init stage (row 307), which only
/// the sandbox sets.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub(crate) init: bool,
}
impl fmt::Debug for SandboxSpec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SandboxSpec")
.field("run", &self.run)
.field("command", &self.command)
.field("timeout_s", &self.timeout_s)
.field("gated", &self.gated)
.field(
"env",
&self.env.iter().map(|(name, _)| name).collect::<Vec<_>>(),
)
.field(
"secrets",
&self
.secrets
.iter()
.map(|(name, _)| name)
.collect::<Vec<_>>(),
)
.field("path", &self.path)
.field("cwd", &self.cwd)
.field("init", &self.init)
.finish()
}
}
/// How one check command ended: `passed` when it exited 0 within its
/// timeout, `timed_out` when the sandbox killed it at its timeout (exit
/// status [`TIMED_OUT`], decision 230). Its last `CHECK_LOG_TAIL_BYTES` of output stay in its run
/// directory's `log`; `tail` is its readable text within
/// `CHECK_OUTPUT_TAIL_BYTES`, the command, its exit status, its failing
/// tests, the lines that look like errors, and the end of the output, with
/// stack frames left out, which a failed action's `action_failed`
/// notifications carry (decisions 193 and 242), and `tests` the failing
/// test names read from the log.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ExecResult {
/// Whether it exited 0 within its timeout.
pub(crate) passed: bool,
/// Whether the sandbox killed it at its timeout.
pub(crate) timed_out: bool,
/// Its readable text, within `CHECK_OUTPUT_TAIL_BYTES` (decision 242).
pub(crate) tail: String,
/// The failing test names read from its output.
pub(crate) tests: Vec<String>,
}
/// The documents the shell reads through `ReadAt` at `frontier`'s heads
/// for a job of `checks`: every file document a check's `paths` or `uses`
/// matches.
pub(crate) fn reads(frontier: &Frontier, checks: &[Action]) -> BTreeSet<String> {
inputs::reads(frontier, checks)
}
/// Write job `job`'s run directories from `files` and give each check's
/// `InputsHash`.
///
/// `files` holds each address the shell read, with its bytes and
/// executable bit. An
/// `InputsHash` is `HASH([check id, paths, uses, {}, toolchain_hash])`, the
/// check id and toolchain hash as 32-byte byte strings; `paths` the map
/// from each file document address the check's `paths` matches to its
/// sorted heads, an array of 32-byte byte strings; `uses` the same for its
/// `uses` matches, so a check reruns whenever a document either set matches
/// moves, a revert to equal bytes included (decision 161); and `{}` the
/// empty dependency frontier. A derived document (one `job.derived`
/// names) maps instead to the CBOR map `{"digest": <SHA-256 of its
/// bytes>}`, so a check reruns only when the bytes it materializes change
/// (draft row A); every other document keeps its heads. A check whose
/// `paths` or `uses` doesn't parse, one of whose documents is missing from
/// `files`, or one reading a derived document whose bytes in `files` don't
/// have the digest `job.derived` gives, is left out, which ends the job
/// unattested.
///
/// # Errors
///
/// When the job's directory, a run directory, or an input file can't be
/// written, or an `InputsHash` can't be encoded.
pub(crate) fn prepare(
job: &Materialize,
files: &BTreeMap<String, TreeFile>,
checks: &[Action],
toolchain_hash: &[u8; 32],
) -> io::Result<BTreeMap<ActionId, InputsHash>> {
inputs::prepare(job, files, checks, toolchain_hash)
}
/// The mode a run gives a file it materializes: 0755 when its document's
/// executable bit is set, else 0644, as the mirror gives it (decision 85,
/// row 307).
const fn file_mode(executable: bool) -> u32 {
if executable { 0o755 } else { 0o644 }
}
/// Run `check`'s steps for job `job` in its run directory through the
/// `run` machine ([`steps`]), journaled as `journal`, recording a `run`
/// span with `outcome` `ok` when it passed and `error` otherwise, a
/// timeout included. Dropping `kill`'s switch cancels it (decision 249,
/// revised).
pub(crate) fn run(
job: &Materialize,
check: &Action,
journal: RunJournal,
kill: KillWatch,
progress: steps::Progress,
) -> ExecResult {
let span = tracing::info_span!("run", outcome = tracing::field::Empty);
let _entered = span.enter();
let profile = Profile {
gated: false,
env: Vec::new(),
withhold: false,
sha: job
.frontier
.id()
.map(|id| crate::ids::hex(&id.0))
.unwrap_or_default(),
kill,
progress,
};
let ran = check.id().ok().and_then(|id| {
steps::execute(&run_dir(&job.dir, &id), check, &profile, journal)
.inspect_err(|err| tracing::error!(error = %err, "a check couldn't run"))
.ok()
});
let passed = ran.as_ref().is_some_and(|ran| ran.passed);
let timed_out = ran.as_ref().is_some_and(|ran| ran.timed_out);
let outcome = if passed { Outcome::Ok } else { Outcome::Error };
span.record("outcome", outcome.as_str());
let (tail, tests) = ran.and_then(|ran| ran.failed).map_or_else(
|| (String::new(), Vec::new()),
|(command, status, log)| summary::summary(&command, status, &log),
);
ExecResult {
passed,
timed_out,
tail,
tests,
}
}
/// The runner's `toolchain_hash` (decisions 121 and 245): the `HASH` of
/// the CBOR array, sorted by value, of `[toolchain, [probe, ...]]` for
/// `rust` and each of `toolchains`, each probe the array `[exit status,
/// stdout bytes]` of its command run in the sandbox on the fixed `PATH`,
/// or the text `absent` when its binary is on none of the fixed `PATH`'s
/// directories.
/// Each probe runs in the fresh run directory `probe/` of the runner
/// directory under `layout`, after `true` has run there to show the
/// sandbox works: an error when it doesn't, so runner mode refuses to
/// start rather than attest under a toolchain hash of failed probes.
pub(crate) fn toolchain_hash(
layout: &Layout,
toolchains: &BTreeSet<Toolchain>,
) -> io::Result<[u8; 32]> {
let probe = runs_dir(layout).join(PROBE_DIR);
inputs::fresh_run_dir(&probe)?;
let (status, _) = probe_check(&probe, &[SANDBOX_CHECK.to_owned()])?;
if status != 0 {
return Err(io::Error::other(format!(
"the sandbox is unavailable here (lvcs sandbox exited {status})"
)));
}
let mut probed: BTreeSet<Toolchain> = toolchains.clone();
probed.insert(Toolchain::Rust);
let mut results = Vec::new();
for toolchain in probed {
let mut probes = Vec::new();
for argv in toolchain.probes() {
let command: Vec<String> = argv.iter().map(|arg| (*arg).to_owned()).collect();
let found = argv.first().is_some_and(|program| {
SANDBOX_PATH
.iter()
.any(|dir| Path::new(dir).join(program).is_file())
});
if !found {
probes.push(Probe::Absent(ABSENT));
continue;
}
inputs::fresh_run_dir(&probe)?;
let (status, stdout) = probe_check(&probe, &command)?;
probes.push(Probe::Ran(status, Bytes(stdout)));
}
results.push((toolchain.as_str(), probes));
}
lvcs_doc::cbor::digest(&results).map_err(|err| io::Error::other(err.to_string()))
}
/// The runs directory `.lvcs/tmp/runs/` of the project at `layout`,
/// which holds every run's directory: each change job's, holding its change
/// actions' run directories, and each gated action's run's.
pub(crate) fn runs_dir(layout: &Layout) -> PathBuf {
layout.tmp_dir().join(RUNS_DIR)
}
/// Job `job`'s directory: `.lvcs/tmp/runs/<job id>/`, the id in
/// `ID_ENCODING`.
pub(crate) fn job_dir(layout: &Layout, job: &JobId) -> PathBuf {
runs_dir(layout).join(ids::encode_id(&job.0))
}
/// Check `check`'s run directory in the job directory `job_dir`:
/// `<job_dir>/<check id>/`, the id in `ID_ENCODING`.
pub(crate) fn run_dir(job_dir: &Path, check: &ActionId) -> PathBuf {
job_dir.join(ids::encode_id(&check.0))
}
/// Make `run` a fresh run directory: 0700, holding empty `work/`,
/// `home/`, `cargo/`, and `tmp/`, its parents created 0700.
///
/// # Errors
///
/// When what is at `run` can't be removed or a directory can't be created.
pub(crate) fn fresh_run_dir(run: &Path) -> io::Result<()> {
inputs::fresh_run_dir(run)
}
/// Remove job directory `job_dir`'s build trees once its job has ended,
/// attested or not: each run directory keeps only its `log`, which stays
/// for the operator (decision 121), so an ended job holds neither its
/// inputs nor its build output on disk. A missing job directory has
/// nothing to remove.
pub(crate) fn prune(job_dir: &Path) -> io::Result<()> {
inputs::prune(job_dir)
}
/// Remove run directory `run`'s build trees once its run has ended,
/// keeping its `log`; a missing one has nothing to remove.
///
/// # Errors
///
/// When a build tree can't be removed.
pub(crate) fn prune_run(run: &Path) -> io::Result<()> {
inputs::prune_run(run)
}
/// `lvcs sandbox`: read a [`SandboxSpec`], one line, from standard input,
/// enter the sandbox, and run its command, exiting with the command's
/// status (128 plus the signal for one a signal ended, 124 after its
/// timeout), 123 when its standard input hung up before the command ended
/// (the step was cancelled), 125 for a spec the sandbox refuses or a
/// sandbox it can't set up, and 2 where Linux namespaces are unavailable.
pub(crate) fn sandbox_main() -> ExitCode {
let mut input = Vec::new();
let spec = io::stdin()
.lock()
.read_until(b'\n', &mut input)
.map_err(|err| err.to_string())
.and_then(|_| serde_json::from_slice::<SandboxSpec>(&input).map_err(|err| err.to_string()));
match spec {
Ok(spec) => enter(&spec),
Err(err) => {
let _ = writeln!(io::stderr(), "lvcs: sandbox: {err}");
ExitCode::from(SANDBOX_REFUSED)
}
}
}