xtask/

artifact.rs

//! The artifact layout: object keys, per-kind provenance, and the manifest.
//!
//! One definition, used by the producers, by finalize and by deploy. Two places
//! that each "know" the layout is how a producer and a consumer drift apart
//! without anyone noticing, which is why this is a module with types rather
//! than a string built in three scripts.
//!
//! Layout:
//!
//! ```text
//! s3://<bucket>/<commit>/<run_id>/
//!     <kind>-<sha256>.tar     the payload, content-addressed
//!     <kind>.meta             pointer + provenance for that kind
//!     nomisync-manifest.json  written by finalize, and only by finalize
//! ```
//!
//! **Content-addressed** because the key contains the hash of the bytes: an
//! object can never be overwritten with different content, since different
//! bytes are a different key. That structurally removes the window where a
//! manifest is valid while the bytes behind a mutable name changed underneath.
//!
//! **Run-scoped, not attempt-scoped.** Attempts may differ between kinds and
//! that is correct: if coverage passes and only release is re-run, they land in
//! attempts 1 and 2 of the same run, built from the same commit. Refusing that
//! pair would force a full coverage rebuild because an unrelated job flaked.
//! The invariant that matters is *same commit, same run, same builder image* —
//! and mixing across runs or commits is unreachable, because the prefix
//! contains both.

use std::fmt;
use std::str::FromStr;

use anyhow::{Context, Result, bail};

pub const MANIFEST_NAME: &str = "nomisync-manifest.json";

/// Which half of the artifact pair an object is.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
    Coverage,
    Release,
}

impl Kind {
    pub const ALL: [Kind; 2] = [Kind::Coverage, Kind::Release];

    pub fn as_str(self) -> &'static str {
        match self {
            Kind::Coverage => "coverage",
            Kind::Release => "release",
        }
    }
}

impl fmt::Display for Kind {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(self.as_str())
    }
}

impl FromStr for Kind {
    type Err = anyhow::Error;

    fn from_str(s: &str) -> Result<Self> {
        match s {
            "coverage" => Ok(Kind::Coverage),
            "release" => Ok(Kind::Release),
            other => bail!("unknown artifact kind: {other}"),
        }
    }
}

/// Who built an artifact, and in which run.
///
/// Read from the environment rather than defaulted. `ci-build` derives Job
/// *names* from `GITHUB_RUN_NUMBER` with a `:-0` fallback, which is right for a
/// name and catastrophic for a path — it would collapse every run onto one
/// prefix and reopen exactly the mixing this layout prevents. A missing value
/// is an error here, and this reads the run id rather than reusing that
/// defaulted number.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Identity {
    pub commit: String,
    pub run_id: String,
    pub run_attempt: String,
    pub builder: String,
}

impl Identity {
    /// Reads identity from the process environment, failing closed.
    pub fn from_env() -> Result<Self> {
        Self::from_source(|key| std::env::var(key).ok())
    }

    /// Reads identity from an arbitrary lookup.
    ///
    /// Taking the source as a parameter keeps the tests off the process
    /// environment: `set_var` is `unsafe` and racy across threads, and cargo
    /// runs a crate's tests in one process.
    pub fn from_source(lookup: impl Fn(&str) -> Option<String>) -> Result<Self> {
        let required = |key: &str| -> Result<String> {
            match lookup(key) {
                Some(value) if !value.is_empty() => Ok(value),
                _ => bail!("{key} is required for artifact paths and was empty or unset"),
            }
        };

        let commit = required("SHA")?;
        // GITHUB_RUN_ID, verified populated in a real build pod (362 while the
        // run number was 12), and globally unique where the number is only
        // sequential per repository. A re-run keeps its id and increments the
        // attempt, which is exactly the prefix behaviour this layout wants.
        let run_id = required("GITHUB_RUN_ID")?;
        let builder = required("RUST_BUILDER_IMAGE_REF")?;

        // The digest requirement lives with the producer, not in the shared
        // ci-build orchestrator: other projects on that orchestrator pin tags
        // and must keep working.
        if !builder.contains("@sha256:") {
            bail!(
                "RUST_BUILDER_IMAGE_REF is not digest-pinned: {builder}\n\
                 a tag can be moved, so it cannot prove two producers used one image"
            );
        }

        Ok(Self {
            commit,
            run_id,
            // Best-effort: unlike the others this is provenance detail, not a
            // path component, so an platform that does not supply it still works.
            run_attempt: lookup("GITHUB_RUN_ATTEMPT")
                .filter(|value| !value.is_empty())
                .unwrap_or_else(|| "unknown".into()),
            builder,
        })
    }

    /// `s3://<bucket>/<commit>/<run_id>` — no trailing slash.
    pub fn prefix(&self, bucket: &str) -> String {
        format!("s3://{bucket}/{}/{}", self.commit, self.run_id)
    }
}

/// A published object: what it is, what it hashes to, and what produced it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Meta {
    pub kind: Kind,
    pub sha256: String,
    pub bytes: u64,
    pub identity: Identity,
}

impl Meta {
    /// Content-addressed key. The hash is in the name, so republishing
    /// different bytes cannot land on the same object.
    pub fn object_key(&self) -> String {
        format!("{}-{}.tar", self.kind, self.sha256)
    }

    pub fn meta_key(kind: Kind) -> String {
        format!("{kind}.meta")
    }

    /// Flat `key=value`, because a meta is read by the deploy container too and
    /// a JSON parser there is a cost with no benefit at this size.
    pub fn to_text(&self) -> String {
        format!(
            "kind={}\nobject={}\nsha256={}\nbytes={}\ncommit={}\nrun_id={}\nrun_attempt={}\nbuilder={}\n",
            self.kind,
            self.object_key(),
            self.sha256,
            self.bytes,
            self.identity.commit,
            self.identity.run_id,
            self.identity.run_attempt,
            self.identity.builder,
        )
    }

    /// Parses a meta fetched from object storage.
    ///
    /// Parsed, never evaluated: the previous shell version had to be careful not
    /// to `source` a file that came off the network. Here that class of mistake
    /// is not expressible.
    pub fn from_text(text: &str) -> Result<Self> {
        let get = |key: &str| -> Result<String> {
            text.lines()
                .find_map(|line| line.strip_prefix(&format!("{key}=")))
                .map(str::to_owned)
                .with_context(|| format!("meta is missing {key}"))
        };

        Ok(Self {
            kind: get("kind")?.parse()?,
            sha256: get("sha256")?,
            bytes: get("bytes")?
                .parse()
                .context("meta bytes is not a number")?,
            identity: Identity {
                commit: get("commit")?,
                run_id: get("run_id")?,
                run_attempt: get("run_attempt")?,
                builder: get("builder")?,
            },
        })
    }
}

/// Why a pair of artifacts may not be assembled into one image.
#[derive(Debug, PartialEq, Eq)]
pub enum PairError {
    Commit { coverage: String, release: String },
    Run { coverage: String, release: String },
    Builder { coverage: String, release: String },
}

impl fmt::Display for PairError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            PairError::Commit { coverage, release } => write!(
                f,
                "artifacts are from different commits: coverage {coverage}, release {release}"
            ),
            PairError::Run { coverage, release } => write!(
                f,
                "artifacts are from different runs: coverage {coverage}, release {release}"
            ),
            PairError::Builder { coverage, release } => write!(
                f,
                "artifacts were built by different images: coverage {coverage}, release {release}"
            ),
        }
    }
}

/// The invariant deploy relies on: same commit, same run, same builder image.
///
/// Attempts deliberately need not match — see the module docs.
pub fn check_pair(coverage: &Meta, release: &Meta) -> Result<(), PairError> {
    let (c, r) = (&coverage.identity, &release.identity);
    if c.commit != r.commit {
        return Err(PairError::Commit {
            coverage: c.commit.clone(),
            release: r.commit.clone(),
        });
    }
    if c.run_id != r.run_id {
        return Err(PairError::Run {
            coverage: c.run_id.clone(),
            release: r.run_id.clone(),
        });
    }
    if c.builder != r.builder {
        return Err(PairError::Builder {
            coverage: c.builder.clone(),
            release: r.builder.clone(),
        });
    }
    Ok(())
}

#[cfg(test)]
mod tests;