xtask/
publish.rs

1//! Publishing an artifact to object storage, and reading one back.
2//!
3//! `s3cmd` stays an external program — it is the credential-handling tool the
4//! platform already provides — but *what* gets published, under which key, and
5//! with which provenance is decided by [`crate::artifact`], in one place.
6
7use std::path::{Path, PathBuf};
8use std::process::Command;
9
10use anyhow::{Context, Result, bail};
11use sha2::{Digest, Sha256};
12
13use crate::artifact::{Identity, Kind, Meta};
14
15/// Credentials and endpoint for the artifacts bucket.
16pub struct Store {
17    access_key: String,
18    secret_key: String,
19    bucket: String,
20    endpoint: String,
21}
22
23impl Store {
24    /// Reads the bucket configuration, failing closed.
25    ///
26    /// Only ever called on a branch where the workflow decided this run may
27    /// hold the write key; a PR run builds and verifies everything and never
28    /// reaches here.
29    pub fn from_env() -> Result<Self> {
30        let get = |key: &str| -> Result<String> {
31            let value = std::env::var(key).unwrap_or_default();
32            if value.is_empty() {
33                bail!("{key} is required to publish artifacts");
34            }
35            Ok(value)
36        };
37        Ok(Self {
38            access_key: get("ARTIFACTS_ACCESS_KEY_ID")?,
39            secret_key: get("ARTIFACTS_SECRET_ACCESS_KEY")?,
40            bucket: get("ARTIFACTS_BUCKET")?,
41            endpoint: get("ARTIFACTS_ENDPOINT")?,
42        })
43    }
44
45    fn s3(&self, args: &[&str]) -> Result<()> {
46        let status = Command::new("s3cmd")
47            .arg(format!("--access_key={}", self.access_key))
48            .arg(format!("--secret_key={}", self.secret_key))
49            .arg(format!("--host={}", self.endpoint))
50            .arg(format!("--host-bucket=%(bucket)s.{}", self.endpoint))
51            .args(args)
52            .status()
53            .context("failed to run s3cmd")?;
54        if !status.success() {
55            bail!("s3cmd {} exited with {status}", args.join(" "));
56        }
57        Ok(())
58    }
59
60    /// Uploads a payload under its content-addressed key, then the meta that
61    /// points at it. Order matters: a meta must never name an object that is
62    /// not there yet, since a reader that sees the meta will go fetch it.
63    pub fn publish(&self, kind: Kind, tarball: &Path, identity: &Identity) -> Result<Meta> {
64        let meta = Meta {
65            kind,
66            sha256: sha256_of(tarball)?,
67            bytes: std::fs::metadata(tarball)
68                .with_context(|| format!("cannot stat {}", tarball.display()))?
69                .len(),
70            identity: identity.clone(),
71        };
72
73        let prefix = identity.prefix(&self.bucket);
74        let object = format!("{prefix}/{}", meta.object_key());
75        println!(">> publishing {} -> {object}", tarball.display());
76        self.s3(&["put", &tarball.to_string_lossy(), &object])?;
77
78        let meta_path = tarball.with_extension("meta");
79        std::fs::write(&meta_path, meta.to_text())
80            .with_context(|| format!("cannot write {}", meta_path.display()))?;
81        self.s3(&[
82            "put",
83            &meta_path.to_string_lossy(),
84            &format!("{prefix}/{}", Meta::meta_key(kind)),
85        ])?;
86
87        Ok(meta)
88    }
89}
90
91/// The hash that becomes part of the object key.
92///
93/// Hex-encoded lowercase, so it is the same string `sha256sum` prints — the
94/// deploy container verifies with that, and a mismatch in encoding would be a
95/// confusing way to discover it.
96pub fn sha256_of(path: &Path) -> Result<String> {
97    use std::io::Read;
98
99    let mut file =
100        std::fs::File::open(path).with_context(|| format!("cannot open {}", path.display()))?;
101    let mut hasher = Sha256::new();
102    let mut buffer = vec![0u8; 64 * 1024];
103    loop {
104        let read = file
105            .read(&mut buffer)
106            .with_context(|| format!("cannot read {}", path.display()))?;
107        if read == 0 {
108            break;
109        }
110        hasher.update(&buffer[..read]);
111    }
112    Ok(hasher
113        .finalize()
114        .iter()
115        .fold(String::with_capacity(64), |mut hex, byte| {
116            use std::fmt::Write;
117            let _ = write!(hex, "{byte:02x}");
118            hex
119        }))
120}
121
122impl Store {
123    fn s3_get(&self, remote: &str, local: &Path) -> Result<()> {
124        self.s3(&["--force", "get", remote, &local.to_string_lossy()])
125    }
126
127    /// Reads one kind's meta out of a run's prefix.
128    pub fn fetch_meta(&self, kind: Kind, identity: &Identity, into: &Path) -> Result<Meta> {
129        let remote = format!("{}/{}", identity.prefix(&self.bucket), Meta::meta_key(kind));
130        let local = into.join(Meta::meta_key(kind));
131        self.s3_get(&remote, &local)
132            .with_context(|| format!("no {kind} meta at {remote} — did that job publish?"))?;
133        let text = std::fs::read_to_string(&local)
134            .with_context(|| format!("cannot read {}", local.display()))?;
135        Meta::from_text(&text).with_context(|| format!("malformed {kind} meta"))
136    }
137
138    /// Downloads a payload and checks it against what its meta claims.
139    ///
140    /// The key is content-addressed, so bytes cannot change under a name — but
141    /// verifying on read is what turns that from an argument into a check.
142    pub fn verify_object(&self, meta: &Meta, identity: &Identity, into: &Path) -> Result<PathBuf> {
143        let remote = format!("{}/{}", identity.prefix(&self.bucket), meta.object_key());
144        let local = into.join(meta.object_key());
145        self.s3_get(&remote, &local)?;
146
147        let bytes = std::fs::metadata(&local)
148            .with_context(|| format!("cannot stat {}", local.display()))?
149            .len();
150        if bytes != meta.bytes {
151            bail!(
152                "{} is {bytes} bytes, its meta says {}",
153                meta.object_key(),
154                meta.bytes
155            );
156        }
157        let sha256 = sha256_of(&local)?;
158        if sha256 != meta.sha256 {
159            bail!(
160                "{} hashes to {sha256}, its meta says {}",
161                meta.object_key(),
162                meta.sha256
163            );
164        }
165        Ok(local)
166    }
167
168    /// Writes the manifest. Only finalize calls this, and only after both kinds
169    /// have been fetched and verified — the manifest existing is the signal that
170    /// the pair is complete and consistent.
171    pub fn write_manifest(&self, metas: &[Meta], identity: &Identity, from: &Path) -> Result<()> {
172        let entries: Vec<serde_json::Value> = metas
173            .iter()
174            .map(|meta| {
175                serde_json::json!({
176                    "kind": meta.kind.as_str(),
177                    "object": meta.object_key(),
178                    "sha256": meta.sha256,
179                    "bytes": meta.bytes,
180                    "run_attempt": meta.identity.run_attempt,
181                })
182            })
183            .collect();
184        let manifest = serde_json::json!({
185            "commit": identity.commit,
186            "run_id": identity.run_id,
187            "builder": identity.builder,
188            "artifacts": entries,
189        });
190
191        let local = from.join(crate::artifact::MANIFEST_NAME);
192        std::fs::write(&local, serde_json::to_vec_pretty(&manifest)?)
193            .with_context(|| format!("cannot write {}", local.display()))?;
194        self.s3(&[
195            "put",
196            &local.to_string_lossy(),
197            &format!(
198                "{}/{}",
199                identity.prefix(&self.bucket),
200                crate::artifact::MANIFEST_NAME
201            ),
202        ])
203    }
204}
xtask/publish.rs

xtask/
publish.rs
