Lines
0 %
Functions
Branches
100 %
//! Publishing an artifact to object storage, and reading one back.
//!
//! `s3cmd` stays an external program — it is the credential-handling tool the
//! platform already provides — but *what* gets published, under which key, and
//! with which provenance is decided by [`crate::artifact`], in one place.
use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{Context, Result, bail};
use sha2::{Digest, Sha256};
use crate::artifact::{Identity, Kind, Meta};
/// Credentials and endpoint for the artifacts bucket.
pub struct Store {
access_key: String,
secret_key: String,
bucket: String,
endpoint: String,
}
impl Store {
/// Reads the bucket configuration, failing closed.
///
/// Only ever called on a branch where the workflow decided this run may
/// hold the write key; a PR run builds and verifies everything and never
/// reaches here.
pub fn from_env() -> Result<Self> {
let get = |key: &str| -> Result<String> {
let value = std::env::var(key).unwrap_or_default();
if value.is_empty() {
bail!("{key} is required to publish artifacts");
Ok(value)
};
Ok(Self {
access_key: get("ARTIFACTS_ACCESS_KEY_ID")?,
secret_key: get("ARTIFACTS_SECRET_ACCESS_KEY")?,
bucket: get("ARTIFACTS_BUCKET")?,
endpoint: get("ARTIFACTS_ENDPOINT")?,
})
fn s3(&self, args: &[&str]) -> Result<()> {
let status = Command::new("s3cmd")
.arg(format!("--access_key={}", self.access_key))
.arg(format!("--secret_key={}", self.secret_key))
.arg(format!("--host={}", self.endpoint))
.arg(format!("--host-bucket=%(bucket)s.{}", self.endpoint))
.args(args)
.status()
.context("failed to run s3cmd")?;
if !status.success() {
bail!("s3cmd {} exited with {status}", args.join(" "));
Ok(())
/// Uploads a payload under its content-addressed key, then the meta that
/// points at it. Order matters: a meta must never name an object that is
/// not there yet, since a reader that sees the meta will go fetch it.
pub fn publish(&self, kind: Kind, tarball: &Path, identity: &Identity) -> Result<Meta> {
let meta = Meta {
kind,
sha256: sha256_of(tarball)?,
bytes: std::fs::metadata(tarball)
.with_context(|| format!("cannot stat {}", tarball.display()))?
.len(),
identity: identity.clone(),
let prefix = identity.prefix(&self.bucket);
let object = format!("{prefix}/{}", meta.object_key());
println!(">> publishing {} -> {object}", tarball.display());
self.s3(&["put", &tarball.to_string_lossy(), &object])?;
let meta_path = tarball.with_extension("meta");
std::fs::write(&meta_path, meta.to_text())
.with_context(|| format!("cannot write {}", meta_path.display()))?;
self.s3(&[
"put",
&meta_path.to_string_lossy(),
&format!("{prefix}/{}", Meta::meta_key(kind)),
])?;
Ok(meta)
/// The hash that becomes part of the object key.
/// Hex-encoded lowercase, so it is the same string `sha256sum` prints — the
/// deploy container verifies with that, and a mismatch in encoding would be a
/// confusing way to discover it.
pub fn sha256_of(path: &Path) -> Result<String> {
use std::io::Read;
let mut file =
std::fs::File::open(path).with_context(|| format!("cannot open {}", path.display()))?;
let mut hasher = Sha256::new();
let mut buffer = vec![0u8; 64 * 1024];
loop {
let read = file
.read(&mut buffer)
.with_context(|| format!("cannot read {}", path.display()))?;
if read == 0 {
break;
hasher.update(&buffer[..read]);
Ok(hasher
.finalize()
.iter()
.fold(String::with_capacity(64), |mut hex, byte| {
use std::fmt::Write;
let _ = write!(hex, "{byte:02x}");
hex
}))
fn s3_get(&self, remote: &str, local: &Path) -> Result<()> {
self.s3(&["--force", "get", remote, &local.to_string_lossy()])
/// Reads one kind's meta out of a run's prefix.
pub fn fetch_meta(&self, kind: Kind, identity: &Identity, into: &Path) -> Result<Meta> {
let remote = format!("{}/{}", identity.prefix(&self.bucket), Meta::meta_key(kind));
let local = into.join(Meta::meta_key(kind));
self.s3_get(&remote, &local)
.with_context(|| format!("no {kind} meta at {remote} — did that job publish?"))?;
let text = std::fs::read_to_string(&local)
.with_context(|| format!("cannot read {}", local.display()))?;
Meta::from_text(&text).with_context(|| format!("malformed {kind} meta"))
/// Downloads a payload and checks it against what its meta claims.
/// The key is content-addressed, so bytes cannot change under a name — but
/// verifying on read is what turns that from an argument into a check.
pub fn verify_object(&self, meta: &Meta, identity: &Identity, into: &Path) -> Result<PathBuf> {
let remote = format!("{}/{}", identity.prefix(&self.bucket), meta.object_key());
let local = into.join(meta.object_key());
self.s3_get(&remote, &local)?;
let bytes = std::fs::metadata(&local)
.with_context(|| format!("cannot stat {}", local.display()))?
.len();
if bytes != meta.bytes {
bail!(
"{} is {bytes} bytes, its meta says {}",
meta.object_key(),
meta.bytes
);
let sha256 = sha256_of(&local)?;
if sha256 != meta.sha256 {
"{} hashes to {sha256}, its meta says {}",
meta.sha256
Ok(local)
/// Writes the manifest. Only finalize calls this, and only after both kinds
/// have been fetched and verified — the manifest existing is the signal that
/// the pair is complete and consistent.
pub fn write_manifest(&self, metas: &[Meta], identity: &Identity, from: &Path) -> Result<()> {
let entries: Vec<serde_json::Value> = metas
.map(|meta| {
serde_json::json!({
"kind": meta.kind.as_str(),
"object": meta.object_key(),
"sha256": meta.sha256,
"bytes": meta.bytes,
"run_attempt": meta.identity.run_attempt,
.collect();
let manifest = serde_json::json!({
"commit": identity.commit,
"run_id": identity.run_id,
"builder": identity.builder,
"artifacts": entries,
});
let local = from.join(crate::artifact::MANIFEST_NAME);
std::fs::write(&local, serde_json::to_vec_pretty(&manifest)?)
.with_context(|| format!("cannot write {}", local.display()))?;
&local.to_string_lossy(),
&format!(
"{}/{}",
identity.prefix(&self.bucket),
crate::artifact::MANIFEST_NAME
),
])