xtask/

release.rs

//! The artifact-producing phases.
//!
//! Ported from `ci/scripts/release-artifact.sh`. The assertions are the part
//! worth keeping: each one exists because something once shipped wrong, and
//! several of them are the only thing standing between a green run and an image
//! that is missing half its content.

use std::path::{Path, PathBuf};
use std::process::Command;

use anyhow::{Context, Result, bail};

use crate::artifact::{Identity, Kind};
use crate::heartbeat::Heartbeat;
use crate::publish::Store;

const DOC_DIR: &str = "web/static/doc";

/// Runs a command, inheriting stdio so its output reaches the job log.
fn run(program: &str, args: &[&str], env: &[(&str, &str)]) -> Result<()> {
    let mut command = Command::new(program);
    command.args(args);
    for (key, value) in env {
        command.env(key, value);
    }
    let status = command
        .status()
        .with_context(|| format!("failed to run {program}"))?;
    if !status.success() {
        bail!("{program} {} exited with {status}", args.join(" "));
    }
    Ok(())
}

/// Instrumented test run plus grcov, producing `web/static/doc/coverage`.
pub fn coverage(heartbeat: &Heartbeat, database_url: &str) -> Result<()> {
    // LLVM_PROFILE_FILE must be unique per process: %p (pid) and %m (binary id)
    // keep the 85 test binaries from overwriting each other's profiles.
    let profraw_dir = std::env::current_dir()?.join("target/coverage");
    let _ = std::fs::remove_dir_all(&profraw_dir);
    std::fs::create_dir_all(&profraw_dir).context("cannot make the profraw dir")?;
    let profile_pattern = profraw_dir.join("nomisync-%p-%m.profraw");

    heartbeat.phase("instrumented test run");
    // Watched, unlike every other phase here: this one prints a line per test,
    // so silence means something is stuck rather than merely slow. A hang here
    // once cost an hour and told us nothing about which test caused it.
    crate::watchdog::run_watched(
        "cargo",
        &["test", "--verbose", "--all-features", "--workspace"],
        &[
            // No -Ccodegen-units=1: LLVM source-based coverage instruments per
            // function, so the CGU count never changed what grcov reports — it
            // only serialised codegen.
            ("RUSTFLAGS", "-Cinstrument-coverage"),
            ("CARGO_INCREMENTAL", "0"),
            ("LLVM_PROFILE_FILE", &profile_pattern.to_string_lossy()),
            ("DATABASE_URL", database_url),
        ],
    )?;

    // A coverage run that produced no profiles yields an empty-but-valid grcov
    // report: green CI, blank coverage page. Assert instead.
    let profraw_count = std::fs::read_dir(&profraw_dir)
        .context("cannot read the profraw dir")?
        .filter_map(Result::ok)
        .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "profraw"))
        .count();
    if profraw_count == 0 {
        bail!("no .profraw produced — coverage instrumentation did not run");
    }
    println!(">> {profraw_count} profraw files");

    heartbeat.phase("grcov");
    // Stale trees must go first: `mv target/doc <existing>` nests the new tree
    // inside the old one, silently pinning published pages to a snapshot.
    let coverage_dir = PathBuf::from(DOC_DIR).join("coverage");
    let _ = std::fs::remove_dir_all(&coverage_dir);
    run(
        "grcov",
        &[
            ".",
            "-s",
            ".",
            "--binary-path",
            "./target/debug/",
            "-t",
            "html",
            "--branch",
            "--ignore-not-existing",
            "--ignore",
            "/*",
            "--ignore",
            "*cranelift*",
            "--ignore",
            "*lexicon*",
            "--ignore",
            "*incremental*",
            "-o",
            &coverage_dir.to_string_lossy(),
        ],
        &[],
    )?;

    // grcov exits zero on an empty profile set, so the tree existing is not the
    // same as the tree having content. This job is the only place that can tell,
    // and it is the artifact deploy will overlay.
    let pages = std::fs::read_dir(&coverage_dir)
        .with_context(|| format!("grcov produced no {}", coverage_dir.display()))?
        .count();
    if pages == 0 {
        bail!(
            "{} is empty — the coverage report has no pages",
            coverage_dir.display()
        );
    }
    println!(">> coverage report: {pages} entries");
    Ok(())
}

/// rustdoc, the org exports, and the static musl binaries.
pub fn build(heartbeat: &Heartbeat, database_url: &str) -> Result<()> {
    let doc_dir = PathBuf::from(DOC_DIR);

    heartbeat.phase("rustdoc");
    for stale in ["rustdoc", "scripts", "nomiscript"] {
        let _ = std::fs::remove_dir_all(doc_dir.join(stale));
    }
    let _ = std::fs::remove_file(doc_dir.join("disdoc.html"));
    // cargo doc recompiles the sqlx macros, so this phase needs a live schema
    // even though it produces no test results.
    run(
        "cargo",
        &["doc", "--workspace", "--all-features", "--no-deps"],
        &[("DATABASE_URL", database_url)],
    )?;
    std::fs::rename("target/doc", doc_dir.join("rustdoc"))
        .context("cannot move the rustdoc tree into the static dir")?;

    heartbeat.phase("org exports");
    std::fs::create_dir_all(doc_dir.join("scripts")).ok();
    std::fs::create_dir_all(doc_dir.join("nomiscript")).ok();
    export_org(Path::new("doc/disdoc.org"))?;
    std::fs::rename("doc/disdoc.html", doc_dir.join("disdoc.html"))
        .context("cannot move disdoc.html")?;
    export_dir(Path::new("doc/scripts"), &doc_dir.join("scripts"))?;
    export_dir(Path::new("doc/nomiscript"), &doc_dir.join("nomiscript"))?;

    heartbeat.phase("musl release build");
    // CXX stays because -sys crates that build C++ need it, and the image has no
    // x86_64-linux-musl-g++ — only the gcc driver.
    run(
        "cargo",
        &[
            "build",
            "--release",
            "--target",
            "x86_64-unknown-linux-musl",
            "-p",
            "web",
            "-p",
            "sshd",
        ],
        &[
            ("SQLX_OFFLINE", "true"),
            ("CC_x86_64_unknown_linux_musl", "x86_64-linux-musl-gcc"),
            ("CXX_x86_64_unknown_linux_musl", "x86_64-linux-musl-gcc"),
        ],
    )?;

    std::fs::copy(
        "target/x86_64-unknown-linux-musl/release/web",
        "release-web",
    )
    .context("cannot stage release-web")?;
    std::fs::copy(
        "target/x86_64-unknown-linux-musl/release/nomisync-sshd",
        "release-sshd",
    )
    .context("cannot stage release-sshd")?;

    verify_binaries()?;
    stage_licences()?;
    Ok(())
}

/// One emacs invocation PER FILE.
///
/// `emacs --batch a.org b.org --eval (org-html-export-to-html)` runs the eval
/// once, against the last visited buffer, so a glob silently publishes a single
/// page — the retired Drone pipeline shipped 1 of 4 script pages exactly that way.
fn export_org(org: &Path) -> Result<()> {
    run(
        "emacs",
        &[
            "-q",
            "--batch",
            &org.to_string_lossy(),
            "-l",
            "ox-html",
            "--eval",
            "(progn (org-babel-do-load-languages (quote org-babel-load-languages) (quote ((emacs-lisp . t) (shell . t)))) (setq org-confirm-babel-evaluate nil) (setq org-html-validation-link nil) (org-html-export-to-html))",
            "-f",
            "save-buffer",
            "-f",
            "kill-emacs",
        ],
        &[],
    )
}

fn export_dir(src: &Path, dest: &Path) -> Result<()> {
    let orgs: Vec<PathBuf> = std::fs::read_dir(src)
        .with_context(|| format!("cannot read {}", src.display()))?
        .filter_map(Result::ok)
        .map(|entry| entry.path())
        .filter(|path| path.extension().is_some_and(|ext| ext == "org"))
        .collect();
    if orgs.is_empty() {
        bail!("no org files in {}", src.display());
    }

    for org in &orgs {
        export_org(org)?;
    }

    // Count what was produced, not what was attempted.
    let produced: Vec<PathBuf> = std::fs::read_dir(src)?
        .filter_map(Result::ok)
        .map(|entry| entry.path())
        .filter(|path| path.extension().is_some_and(|ext| ext == "html"))
        .collect();
    if produced.len() != orgs.len() {
        bail!(
            "{}: exported {} of {} pages",
            src.display(),
            produced.len(),
            orgs.len()
        );
    }
    for page in produced {
        let target = dest.join(page.file_name().context("html page has no file name")?);
        std::fs::rename(&page, &target)
            .with_context(|| format!("cannot move {}", page.display()))?;
    }
    println!(">> exported {} pages from {}", orgs.len(), src.display());
    Ok(())
}

/// The assertions that prove the shipped binaries are what we think they are.
fn verify_binaries() -> Result<()> {
    for binary in ["release-web", "release-sshd"] {
        let bytes = std::fs::read(binary).with_context(|| format!("cannot read {binary}"))?;

        // Coverage instrumentation in a shipped binary is invisible until
        // production is slow and writing .profraw files.
        if contains(&bytes, b"__llvm_profile") {
            bail!(
                "{binary} contains coverage instrumentation — RUSTFLAGS leaked out of the coverage phase"
            );
        }

        // plotters prefers its `ttf` backend when both are enabled, and that
        // backend dlopens fontconfig — impossible in a static musl binary, so
        // chart rendering PANICS rather than degrading. The builder still
        // carries the fontconfig packages, which would let such a regression
        // build cleanly and only fail inside the scratch image.
        if binary == "release-web" && contains(&bytes, b"fontconfig") {
            bail!(
                "release-web references fontconfig — plotters' ttf backend is back; it cannot work in the scratch image"
            );
        }
    }

    // The generated trees are gitignored, so their absence here means a phase
    // above failed quietly. The deploy job asserts the same on the unpacked side.
    //
    // NOT coverage: that is the coverage job's output, published as its own
    // artifact and overlaid at deploy time. Asserting it here is what made the
    // first xtask release run fail — the phase had moved, the assertion had not.
    for required in ["rustdoc", "scripts", "nomiscript", "disdoc.html"] {
        let path = PathBuf::from(DOC_DIR).join(required);
        if !path.exists() {
            bail!("missing {}", path.display());
        }
    }
    Ok(())
}

fn contains(haystack: &[u8], needle: &[u8]) -> bool {
    haystack
        .windows(needle.len())
        .any(|window| window == needle)
}

/// release-web embeds DejaVuSans via include_bytes!, and that font's licence
/// requires its notice to travel with copies of the font software. web/static is
/// copied wholesale into the runtime image, so putting it here makes it ship
/// with both the tarball and the image, and be servable.
fn stage_licences() -> Result<()> {
    let dir = PathBuf::from("web/static/licenses");
    std::fs::create_dir_all(&dir).context("cannot make the licenses dir")?;
    let target = dir.join("DejaVuSans-LICENSE.txt");
    std::fs::copy("plotting/fonts/LICENSE", &target).context("cannot copy the font licence")?;
    if std::fs::metadata(&target)?.len() == 0 {
        bail!("font licence missing from artifact");
    }
    Ok(())
}

/// Tars the payload for a kind and, when this run holds the write key, publishes it.
pub fn package(kind: Kind, heartbeat: &Heartbeat, with_artifacts: bool) -> Result<()> {
    heartbeat.phase("packaging");

    let tarball = PathBuf::from(format!("nomisync-{kind}.tar"));
    let members: &[&str] = match kind {
        // Only the coverage tree, so the two payloads are disjoint by rule and
        // the assembler can reject any overlap rather than guess.
        Kind::Coverage => &["web/static/doc/coverage"],
        Kind::Release => &["release-web", "release-sshd", "web/static"],
    };
    let mut args = vec!["-czf", &tarball.to_str().context("tarball path")?];
    args.extend_from_slice(members);
    run("tar", &args, &[])?;

    if !with_artifacts {
        println!(">> WITH_ARTIFACTS unset — built and verified, not uploading");
        return Ok(());
    }

    let identity = Identity::from_env()?;
    let store = Store::from_env()?;
    let meta = store.publish(kind, &tarball, &identity)?;
    println!(
        ">> published {} ({} bytes) for {} run {}",
        meta.object_key(),
        meta.bytes,
        identity.commit,
        identity.run_id
    );
    Ok(())
}