derive-exec

Source: crates/lvcs/src/runner/derive.rs.

/// One derivation run (row 180) of runner job `job`.
///
/// It regenerates the derived files from the documents `inputs` names at
/// `frontier`, running the generator `action`'s steps in the run directory
/// `<dir>/derive/`, where `dir` is the job's directory under
/// `.lvcs/tmp/runs/`, so `lvcs sandbox` gives it a check's profile.
/// `frontier` is the job's frontier less the documents deleted at their
/// heads there, as a check's [`super::Materialize`] is: an input it doesn't
/// hold is absent from the run.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DeriveJob {
    /// The runner job the run belongs to.
    pub(crate) job: JobId,
    /// The frontier it runs on.
    pub(crate) frontier: Frontier,
    /// The job's directory.
    pub(crate) dir: PathBuf,
    /// The generator.
    pub(crate) action: Action,
    /// The input documents' addresses.
    pub(crate) inputs: BTreeSet<String>,
}

/// How a derivation run ended.
///
/// It ends with the outputs it wrote, each derived file it created or
/// changed mapped to its bytes and their SHA-256 (sorted, at most
/// `GENERATION_OUTPUTS_MAX`), or why it failed. The runner machine's
/// own outcome type is lvcs-action's; the daemon maps this to it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum DeriveResult {
    /// It exited 0 and wrote only derived files.
    Outputs(BTreeMap<String, Output>),
    /// It exited non-zero, was killed, or timed out (`Exit`), or it
    /// created, changed, or removed a file `is_derived` doesn't call
    /// derived (`NonDerivedWrite`).
    Failed(DeriveFailure),
}

/// One output of a derivation: its bytes and their SHA-256.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Output {
    /// The SHA-256 of its bytes.
    pub(crate) digest: [u8; 32],
    /// Its bytes, which the runner writes into its document.
    pub(crate) bytes: Vec<u8>,
}

/// Run derivation `job` on `files`, classifying what it touched.
///
/// `files` holds the bytes and executable bit the shell read for its
/// inputs at its frontier, and each file the run touched is classified with `is_derived`, which
/// takes the file's path and its first
/// `DERIVED_HEADER_BYTES` (a removed file's from before the run). A path
/// that isn't an input and that `syncs` says the project never syncs (a
/// `.gitignore`d or `.lvcsignore`d one, or never-sync) is left out of the
/// walk whatever it is, so it is never an output (decision 299).
///
/// The run directory is made fresh (0700, empty `work/`, `home/`,
/// `cargo/`, and `tmp/`), and `work/` holds each input `frontier` holds,
/// with its bytes from `files`, mode 0755 when its document's executable
/// bit is set and 0644 otherwise (row 307). The generator's steps run there through
/// the `run` machine, journaled as `journal`, each through `lvcs sandbox`
/// with a change action's profile: no network, no secrets, within its
/// `timeout_secs`. A run that doesn't pass (a step's non-zero exit, a
/// kill, a timeout, or bad step files) is `Failed(Exit)`. Otherwise every regular file under `work/` that is new
/// or whose bytes changed, and every input that is gone, is classified: a
/// derived one that is new or changed is an output, and any other is
/// `Failed(NonDerivedWrite)`, as is anything under `work/` that is neither
/// a regular file nor a directory (a symlink, a device, a FIFO, a socket)
/// or whose path isn't UTF-8. The run directory ends with its `log` alone:
/// `work/`, `home/`, `cargo/`, and `tmp/` are removed whatever happened.
/// The run records a `derive` span whose `outcome` is `ok` for outputs and
/// `error` otherwise. Dropping `kill`'s switch cancels the run (decision
/// 249, revised): its step under way is killed, and it is `Failed(Exit)`.
///
/// # Errors
///
/// With no result, which the runner must not record: an input
/// `frontier` holds is missing from `files` (a refused read) or isn't a
/// plain relative path; the run directory can't be made, written, walked,
/// or cleared; the sandbox can't be started; or the run's result can't be
/// recorded as a generation (a derived input it removed, which has no
/// bytes to name; more than `GENERATION_OUTPUTS_MAX` outputs; or a file
/// over `CHECK_FILE_BYTES`, which the sandbox's file size limit forbids).
pub(crate) fn derive(
    job: &DeriveJob,
    files: &BTreeMap<String, TreeFile>,
    is_derived: &dyn Fn(&str, &[u8]) -> bool,
    syncs: &dyn Fn(&str) -> bool,
    journal: RunJournal,
    kill: KillWatch,
) -> io::Result<DeriveResult> {
    let span = tracing::info_span!("derive", outcome = Empty);
    let _entered = span.enter();
    let result = run_derivation(job, files, is_derived, syncs, journal, kill);
    let outcome = match &result {
        Ok(DeriveResult::Outputs(_)) => Outcome::Ok,
        Ok(DeriveResult::Failed(_)) | Err(_) => Outcome::Error,
    };
    span.record("outcome", outcome.as_str());
    result
}
/// The input documents of a derivation at `frontier` (row 180).
///
/// Its default inputs are every file document `frontier` holds that isn't
/// derived (decision 171): `derived` names those whose `/meta/derived` is
/// true at their heads there, as the shell read them. `[derive] inputs`
/// patterns, when set, narrow the default, so a derived document is never
/// an input. `None` when a pattern doesn't parse, which the config's parse
/// already refuses.
pub(crate) fn derive_inputs(
    frontier: &Frontier,
    derived: &BTreeSet<String>,
    patterns: Option<&[String]>,
) -> Option<BTreeSet<String>> {
    let narrow = match patterns {
        Some(given) => Some(PathSet::new(given, false).ok()?),
        None => None,
    };
    Some(
        inputs::file_docs(frontier)
            .filter(|(address, _)| !derived.contains(*address))
            .filter(|(address, _)| narrow.as_ref().is_none_or(|set| set.matches(address)))
            .map(|(address, _)| address.clone())
            .collect(),
    )
}

/// What a derivation's `InputsHash` hashes: `[derive id, inputs,
/// toolchain_hash]`, `inputs` mapping each input document to its sorted
/// heads.
#[derive(Serialize)]
struct DeriveHashed<'a>(
    &'a ActionId,
    BTreeMap<&'a str, inputs::Heads>,
    #[serde(with = "lvcs_doc::cbor::fixed")] [u8; 32],
);

/// Derivation `derive`'s `InputsHash` at `frontier` (row 180).
///
/// It is `HASH([derive id, inputs, toolchain_hash])`, the derive id and the
/// toolchain hash as 32-byte byte strings, and `inputs` the map from each
/// address of `inputs` that `frontier` holds to its sorted heads, an array
/// of 32-byte byte strings, as a check's `InputsHash` maps a document that
/// isn't derived (decision 161).
///
/// # Errors
///
/// When the hashed array can't be encoded.
pub(crate) fn derive_inputs_hash(
    derive: &ActionId,
    frontier: &Frontier,
    inputs: &BTreeSet<String>,
    toolchain_hash: &[u8; 32],
) -> io::Result<InputsHash> {
    let heads: BTreeMap<&str, inputs::Heads> = frontier
        .docs
        .iter()
        .filter(|(address, _)| inputs.contains(*address))
        .map(|(address, heads)| (address.as_str(), inputs::Heads::of(heads)))
        .collect();
    cbor::digest(&DeriveHashed(derive, heads, *toolchain_hash))
        .map(InputsHash)
        .map_err(|err| io::Error::other(err.to_string()))
}