Lines
83.51 %
Functions
78.95 %
Branches
100 %
//! The artifact layout: object keys, per-kind provenance, and the manifest.
//!
//! One definition, used by the producers, by finalize and by deploy. Two places
//! that each "know" the layout is how a producer and a consumer drift apart
//! without anyone noticing, which is why this is a module with types rather
//! than a string built in three scripts.
//! Layout:
//! ```text
//! s3://<bucket>/<commit>/<run_id>/
//! <kind>-<sha256>.tar the payload, content-addressed
//! <kind>.meta pointer + provenance for that kind
//! nomisync-manifest.json written by finalize, and only by finalize
//! ```
//! **Content-addressed** because the key contains the hash of the bytes: an
//! object can never be overwritten with different content, since different
//! bytes are a different key. That structurally removes the window where a
//! manifest is valid while the bytes behind a mutable name changed underneath.
//! **Run-scoped, not attempt-scoped.** Attempts may differ between kinds and
//! that is correct: if coverage passes and only release is re-run, they land in
//! attempts 1 and 2 of the same run, built from the same commit. Refusing that
//! pair would force a full coverage rebuild because an unrelated job flaked.
//! The invariant that matters is *same commit, same run, same builder image* —
//! and mixing across runs or commits is unreachable, because the prefix
//! contains both.
use std::fmt;
use std::str::FromStr;
use anyhow::{Context, Result, bail};
pub const MANIFEST_NAME: &str = "nomisync-manifest.json";
/// Which half of the artifact pair an object is.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Kind {
Coverage,
Release,
}
impl Kind {
pub const ALL: [Kind; 2] = [Kind::Coverage, Kind::Release];
pub fn as_str(self) -> &'static str {
match self {
Kind::Coverage => "coverage",
Kind::Release => "release",
impl fmt::Display for Kind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
impl FromStr for Kind {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
match s {
"coverage" => Ok(Kind::Coverage),
"release" => Ok(Kind::Release),
other => bail!("unknown artifact kind: {other}"),
/// Who built an artifact, and in which run.
///
/// Read from the environment rather than defaulted. `ci-build` derives Job
/// *names* from `GITHUB_RUN_NUMBER` with a `:-0` fallback, which is right for a
/// name and catastrophic for a path — it would collapse every run onto one
/// prefix and reopen exactly the mixing this layout prevents. A missing value
/// is an error here, and this reads the run id rather than reusing that
/// defaulted number.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Identity {
pub commit: String,
pub run_id: String,
pub run_attempt: String,
pub builder: String,
impl Identity {
/// Reads identity from the process environment, failing closed.
pub fn from_env() -> Result<Self> {
Self::from_source(|key| std::env::var(key).ok())
/// Reads identity from an arbitrary lookup.
/// Taking the source as a parameter keeps the tests off the process
/// environment: `set_var` is `unsafe` and racy across threads, and cargo
/// runs a crate's tests in one process.
pub fn from_source(lookup: impl Fn(&str) -> Option<String>) -> Result<Self> {
let required = |key: &str| -> Result<String> {
match lookup(key) {
Some(value) if !value.is_empty() => Ok(value),
_ => bail!("{key} is required for artifact paths and was empty or unset"),
};
let commit = required("SHA")?;
// GITHUB_RUN_ID, verified populated in a real build pod (362 while the
// run number was 12), and globally unique where the number is only
// sequential per repository. A re-run keeps its id and increments the
// attempt, which is exactly the prefix behaviour this layout wants.
let run_id = required("GITHUB_RUN_ID")?;
let builder = required("RUST_BUILDER_IMAGE_REF")?;
// The digest requirement lives with the producer, not in the shared
// ci-build orchestrator: other projects on that orchestrator pin tags
// and must keep working.
if !builder.contains("@sha256:") {
bail!(
"RUST_BUILDER_IMAGE_REF is not digest-pinned: {builder}\n\
a tag can be moved, so it cannot prove two producers used one image"
);
Ok(Self {
commit,
run_id,
// Best-effort: unlike the others this is provenance detail, not a
// path component, so an platform that does not supply it still works.
run_attempt: lookup("GITHUB_RUN_ATTEMPT")
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "unknown".into()),
builder,
})
/// `s3://<bucket>/<commit>/<run_id>` — no trailing slash.
pub fn prefix(&self, bucket: &str) -> String {
format!("s3://{bucket}/{}/{}", self.commit, self.run_id)
/// A published object: what it is, what it hashes to, and what produced it.
pub struct Meta {
pub kind: Kind,
pub sha256: String,
pub bytes: u64,
pub identity: Identity,
impl Meta {
/// Content-addressed key. The hash is in the name, so republishing
/// different bytes cannot land on the same object.
pub fn object_key(&self) -> String {
format!("{}-{}.tar", self.kind, self.sha256)
pub fn meta_key(kind: Kind) -> String {
format!("{kind}.meta")
/// Flat `key=value`, because a meta is read by the deploy container too and
/// a JSON parser there is a cost with no benefit at this size.
pub fn to_text(&self) -> String {
format!(
"kind={}\nobject={}\nsha256={}\nbytes={}\ncommit={}\nrun_id={}\nrun_attempt={}\nbuilder={}\n",
self.kind,
self.object_key(),
self.sha256,
self.bytes,
self.identity.commit,
self.identity.run_id,
self.identity.run_attempt,
self.identity.builder,
)
/// Parses a meta fetched from object storage.
/// Parsed, never evaluated: the previous shell version had to be careful not
/// to `source` a file that came off the network. Here that class of mistake
/// is not expressible.
pub fn from_text(text: &str) -> Result<Self> {
let get = |key: &str| -> Result<String> {
text.lines()
.find_map(|line| line.strip_prefix(&format!("{key}=")))
.map(str::to_owned)
.with_context(|| format!("meta is missing {key}"))
kind: get("kind")?.parse()?,
sha256: get("sha256")?,
bytes: get("bytes")?
.parse()
.context("meta bytes is not a number")?,
identity: Identity {
commit: get("commit")?,
run_id: get("run_id")?,
run_attempt: get("run_attempt")?,
builder: get("builder")?,
},
/// Why a pair of artifacts may not be assembled into one image.
#[derive(Debug, PartialEq, Eq)]
pub enum PairError {
Commit { coverage: String, release: String },
Run { coverage: String, release: String },
Builder { coverage: String, release: String },
impl fmt::Display for PairError {
PairError::Commit { coverage, release } => write!(
f,
"artifacts are from different commits: coverage {coverage}, release {release}"
),
PairError::Run { coverage, release } => write!(
"artifacts are from different runs: coverage {coverage}, release {release}"
PairError::Builder { coverage, release } => write!(
"artifacts were built by different images: coverage {coverage}, release {release}"
/// The invariant deploy relies on: same commit, same run, same builder image.
/// Attempts deliberately need not match — see the module docs.
pub fn check_pair(coverage: &Meta, release: &Meta) -> Result<(), PairError> {
let (c, r) = (&coverage.identity, &release.identity);
if c.commit != r.commit {
return Err(PairError::Commit {
coverage: c.commit.clone(),
release: r.commit.clone(),
});
if c.run_id != r.run_id {
return Err(PairError::Run {
coverage: c.run_id.clone(),
release: r.run_id.clone(),
if c.builder != r.builder {
return Err(PairError::Builder {
coverage: c.builder.clone(),
release: r.builder.clone(),
Ok(())
#[cfg(test)]
mod tests;