xtask/
coverage.rs

1//! The coverage phase: an instrumented test run rendered into
2//! `web/static/doc/coverage`.
3//!
4//! grcov can read the raw profiles itself, but it then holds every binary's
5//! results at once and runs one `llvm-cov` per CPU — about 13 GiB for this
6//! workspace, enough to OOM-kill the job on an 8 GiB CI node. So the profiles
7//! are merged here single-threaded, each instrumented object is exported to lcov
8//! on its own, and grcov only renders. Exporting per object keeps grcov's
9//! numbers: one export over every object discards the counters of functions
10//! compiled differently in different binaries.
11//!
12//! Unlike grcov, which skips an object llvm-cov cannot export, a failed export
13//! fails the phase: every exported object carries a real coverage map, so a
14//! failure means a broken build, and a quietly lower percentage would hide it.
15
16use std::ffi::OsStr;
17use std::fs::File;
18use std::io::{self, BufRead, BufReader, BufWriter, ErrorKind, Read, Seek, SeekFrom, Write};
19use std::os::unix::fs::PermissionsExt;
20use std::path::{Path, PathBuf};
21use std::process::{Command, Stdio};
22
23use anyhow::{Context, Result, bail};
24
25use crate::clean::remove_dir_if_present;
26use crate::heartbeat::Heartbeat;
27use crate::release::{DOC_DIR, run};
28
29const PROFRAW_DIR: &str = "target/coverage";
30const PROFDATA: &str = "target/coverage.profdata";
31const LCOV_DIR: &str = "target/coverage-lcov";
32const TARGET_DIR: &str = "target/debug";
33/// The section every instrumented object carries.
34const COVMAP_SECTION: &[u8] = b"__llvm_covmap";
35
36/// ELF magic, 64-bit class, little-endian data: the only objects CI builds.
37const ELF_IDENT: &[u8] = b"\x7fELF\x02\x01";
38const ELF_HEADER_LEN: usize = 64;
39const E_SHOFF: usize = 0x28;
40const E_SHENTSIZE: usize = 0x3a;
41const E_SHNUM: usize = 0x3c;
42const E_SHSTRNDX: usize = 0x3e;
43const SH_NAME: usize = 0x00;
44const SH_OFFSET: usize = 0x18;
45const SH_SIZE: usize = 0x20;
46const SH_LINK: usize = 0x28;
47/// The name-table index an object with extended numbering stores in section 0.
48const SHN_XINDEX: usize = 0xffff;
49/// Real section and name tables are kilobytes; anything past this is corrupt.
50const MAX_TABLE_LEN: u64 = 16 << 20;
51
52/// Instrumented test run, then the HTML report.
53pub fn run_coverage(heartbeat: &Heartbeat, database_url: &str) -> Result<()> {
54    let cwd = std::env::current_dir()?;
55    let profraw_dir = cwd.join(PROFRAW_DIR);
56    instrumented_test_run(heartbeat, &profraw_dir, database_url)?;
57
58    let llvm_bin = llvm_bin()?;
59    let profdata = Path::new(PROFDATA);
60    heartbeat.phase("merge profiles");
61    merge_profiles(&llvm_bin.join("llvm-profdata"), &profraw_dir, profdata)?;
62
63    heartbeat.phase("llvm-cov export");
64    let exported = export_objects(
65        &llvm_bin.join("llvm-cov"),
66        profdata,
67        Path::new(TARGET_DIR),
68        Path::new(LCOV_DIR),
69        &format!("{}/", cwd.display()),
70    )?;
71    println!(">> exported {exported} instrumented objects");
72
73    heartbeat.phase("grcov");
74    render_html(
75        "grcov",
76        Path::new(LCOV_DIR),
77        &PathBuf::from(DOC_DIR).join("coverage"),
78    )
79}
80
81fn instrumented_test_run(
82    heartbeat: &Heartbeat,
83    profraw_dir: &Path,
84    database_url: &str,
85) -> Result<()> {
86    // LLVM_PROFILE_FILE must be unique per process: %p (pid) and %m (binary id)
87    // keep the 85 test binaries from overwriting each other's profiles.
88    remove_dir_if_present(profraw_dir)?;
89    std::fs::create_dir_all(profraw_dir).context("cannot make the profraw dir")?;
90    let profile_pattern = profraw_dir.join("nomisync-%p-%m.profraw");
91
92    heartbeat.phase("instrumented test run");
93    // Watched, unlike every other phase here: this one prints a line per test,
94    // so silence means something is stuck rather than merely slow. A hang here
95    // once cost an hour and told us nothing about which test caused it.
96    crate::watchdog::run_watched(
97        "cargo",
98        &["test", "--verbose", "--all-features", "--workspace"],
99        &[
100            // No -Ccodegen-units=1: LLVM source-based coverage instruments per
101            // function, so the CGU count never changed what grcov reports — it
102            // only serialised codegen.
103            ("RUSTFLAGS", "-Cinstrument-coverage"),
104            ("CARGO_INCREMENTAL", "0"),
105            ("LLVM_PROFILE_FILE", &profile_pattern.to_string_lossy()),
106            ("DATABASE_URL", database_url),
107        ],
108    )?;
109
110    // A coverage run that produced no profiles yields an empty-but-valid grcov
111    // report: green CI, blank coverage page. Assert instead.
112    let profraw_count = std::fs::read_dir(profraw_dir)
113        .context("cannot read the profraw dir")?
114        .filter_map(Result::ok)
115        .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "profraw"))
116        .count();
117    if profraw_count == 0 {
118        bail!("no .profraw produced — coverage instrumentation did not run");
119    }
120    println!(">> {profraw_count} profraw files");
121    Ok(())
122}
123
124/// `llvm-tools` installs its binaries beside the host target's libdir.
125fn llvm_bin() -> Result<PathBuf> {
126    let output = Command::new("rustc")
127        .args(["--print", "target-libdir"])
128        .output()
129        .context("failed to run rustc")?;
130    if !output.status.success() {
131        bail!("rustc --print target-libdir exited with {}", output.status);
132    }
133    let libdir = String::from_utf8(output.stdout).context("rustc printed a non-UTF-8 libdir")?;
134    let bin =
135        llvm_bin_for(Path::new(libdir.trim())).context("rustc's target libdir has no parent")?;
136    require_llvm_tools(bin)
137}
138
139fn llvm_bin_for(libdir: &Path) -> Option<PathBuf> {
140    libdir.parent().map(|host| host.join("bin"))
141}
142
143fn require_llvm_tools(bin: PathBuf) -> Result<PathBuf> {
144    if !bin.join("llvm-profdata").is_file() {
145        bail!(
146            "no llvm-profdata in {} — install it with `rustup component add llvm-tools`",
147            bin.display()
148        );
149    }
150    Ok(bin)
151}
152
153/// One thread: the default of one per CPU took 3 GiB for this merge, one
154/// thread takes 250 MiB.
155fn merge_profiles(tool: &Path, profraw_dir: &Path, profdata: &Path) -> Result<()> {
156    run(
157        &tool.to_string_lossy(),
158        &[
159            "merge",
160            "-sparse",
161            "--num-threads=1",
162            "-o",
163            &profdata.to_string_lossy(),
164            &profraw_dir.to_string_lossy(),
165        ],
166        &[],
167    )
168}
169
170fn export_objects(
171    tool: &Path,
172    profdata: &Path,
173    target_dir: &Path,
174    lcov_dir: &Path,
175    root: &str,
176) -> Result<usize> {
177    let objects = instrumented_objects(target_dir)?;
178    if objects.is_empty() {
179        bail!("no instrumented objects under {}", target_dir.display());
180    }
181    remove_dir_if_present(lcov_dir)?;
182    std::fs::create_dir_all(lcov_dir).context("cannot make the lcov dir")?;
183
184    let mut workspace_sources = 0usize;
185    for (index, object) in objects.iter().enumerate() {
186        let info = lcov_dir.join(format!("{index}.info"));
187        let kept = export_object(tool, profdata, object, &info, root)?;
188        workspace_sources = workspace_sources.saturating_add(kept);
189    }
190    // grcov still writes an index and badges when every source was ignored, so
191    // a checkout path llvm-cov does not report would otherwise publish a green,
192    // empty report.
193    if workspace_sources == 0 {
194        bail!("no lcov source path starts with {root}; the report would have no workspace files");
195    }
196    Ok(objects.len())
197}
198
199/// Returns how many workspace source records were kept.
200fn export_object(
201    tool: &Path,
202    profdata: &Path,
203    object: &Path,
204    info: &Path,
205    root: &str,
206) -> Result<usize> {
207    let mut child = Command::new(tool)
208        .args(["export", "-format=lcov", "-instr-profile"])
209        .arg(profdata)
210        .arg(object)
211        .stdout(Stdio::piped())
212        .spawn()
213        .with_context(|| format!("failed to run {}", tool.display()))?;
214    let stdout = child
215        .stdout
216        .take()
217        .context("llvm-cov stdout was not captured")?;
218    // The reader is dropped before the wait, so a failed copy closes the pipe
219    // and llvm-cov exits instead of blocking on a full pipe.
220    let copied = copy_workspace_records(stdout, info, root);
221    let status = child
222        .wait()
223        .with_context(|| format!("llvm-cov export {} did not finish", object.display()))?;
224    let kept = copied?;
225    if !status.success() {
226        bail!("llvm-cov export {} exited with {status}", object.display());
227    }
228    Ok(kept)
229}
230
231/// Copies only the records of sources under `root`, with their paths made
232/// relative. The report ignores every absolute path anyway, and dropping the
233/// registry and toolchain records here keeps gigabytes of lcov off the disk.
234/// Returns how many records were kept.
235fn copy_workspace_records(lcov: impl Read, info: &Path, root: &str) -> Result<usize> {
236    let file = File::create(info).with_context(|| format!("cannot create {}", info.display()))?;
237    let mut writer = BufWriter::new(file);
238    let mut kept = 0usize;
239    let mut keep = false;
240    for line in BufReader::new(lcov).lines() {
241        let line = line?;
242        match line
243            .strip_prefix("SF:")
244            .map(|source| source.strip_prefix(root))
245        {
246            Some(Some(relative)) => {
247                keep = true;
248                kept = kept.saturating_add(1);
249                writeln!(writer, "SF:{relative}")?;
250            }
251            Some(None) => keep = false,
252            None if keep => writeln!(writer, "{line}")?,
253            None => {}
254        }
255    }
256    writer
257        .flush()
258        .with_context(|| format!("cannot write {}", info.display()))?;
259    Ok(kept)
260}
261
262fn instrumented_objects(dir: &Path) -> Result<Vec<PathBuf>> {
263    let mut found = Vec::new();
264    collect_objects(dir, &mut found)?;
265    found.sort();
266    Ok(found)
267}
268
269fn collect_objects(dir: &Path, found: &mut Vec<PathBuf>) -> Result<()> {
270    let entries =
271        std::fs::read_dir(dir).with_context(|| format!("cannot read {}", dir.display()))?;
272    for entry in entries {
273        let entry = entry?;
274        let path = entry.path();
275        let file_type = entry.file_type()?;
276        if file_type.is_dir() {
277            collect_objects(&path, found)?;
278        } else if file_type.is_file()
279            && is_candidate(&path, entry.metadata()?.permissions().mode())
280            && has_coverage_map(&path)?
281        {
282            found.push(path);
283        }
284    }
285    Ok(())
286}
287
288/// Executables (tests, binaries, build scripts) and shared objects (proc
289/// macros, which run inside rustc) are what carry coverage maps.
290fn is_candidate(path: &Path, mode: u32) -> bool {
291    match path.extension().and_then(OsStr::to_str) {
292        Some("so") => true,
293        Some("d") => false,
294        _ => mode & 0o111 != 0,
295    }
296}
297
298fn has_coverage_map(path: &Path) -> Result<bool> {
299    let mut file = File::open(path).with_context(|| format!("cannot open {}", path.display()))?;
300    elf_has_section(&mut file, COVMAP_SECTION)
301        .with_context(|| format!("cannot read {}", path.display()))
302}
303
304/// Looks the name up in the ELF section table. Matching it anywhere in the
305/// file also accepts binaries that merely contain the string — xtask itself
306/// among them — and llvm-cov refuses to export those.
307fn elf_has_section(file: &mut (impl Read + Seek), name: &[u8]) -> io::Result<bool> {
308    let mut header = [0; ELF_HEADER_LEN];
309    if !fill(file, &mut header)? {
310        return Ok(false);
311    }
312    let Some(sections) = SectionTable::parse(&header) else {
313        return Ok(false);
314    };
315    let Some(sections) = resolve_extended(file, sections)? else {
316        return Ok(false);
317    };
318    let Some(table) = read_region(file, sections.offset, sections.len())? else {
319        return Ok(false);
320    };
321    let Some(names_entry) = sections.entry(&table, sections.names_index) else {
322        return Ok(false);
323    };
324    let (Some(names_offset), Some(names_len)) = (
325        le_bytes(names_entry, SH_OFFSET).map(u64::from_le_bytes),
326        le_bytes(names_entry, SH_SIZE).map(u64::from_le_bytes),
327    ) else {
328        return Ok(false);
329    };
330    let Some(names) = read_region(file, names_offset, usize::try_from(names_len).ok())? else {
331        return Ok(false);
332    };
333    Ok((0..sections.count)
334        .filter_map(|index| sections.entry(&table, index))
335        .filter_map(|entry| le_bytes(entry, SH_NAME).map(u32::from_le_bytes))
336        .any(|name_offset| section_name(&names, name_offset) == Some(name)))
337}
338
339/// With 0xff00 or more sections the header's count is 0 and its name index is
340/// SHN_XINDEX; the real values live in section 0's `sh_size` and `sh_link`.
341fn resolve_extended(
342    file: &mut (impl Read + Seek),
343    sections: SectionTable,
344) -> io::Result<Option<SectionTable>> {
345    if sections.count != 0 && sections.names_index != SHN_XINDEX {
346        return Ok(Some(sections));
347    }
348    if sections.offset == 0 {
349        return Ok(None);
350    }
351    let Some(first) = read_region(file, sections.offset, Some(sections.entry_len))? else {
352        return Ok(None);
353    };
354    let count = match sections.count {
355        0 => le_bytes(&first, SH_SIZE)
356            .map(u64::from_le_bytes)
357            .and_then(|size| usize::try_from(size).ok()),
358        count => Some(count),
359    };
360    let names_index = match sections.names_index {
361        SHN_XINDEX => le_bytes(&first, SH_LINK)
362            .map(u32::from_le_bytes)
363            .and_then(|link| usize::try_from(link).ok()),
364        index => Some(index),
365    };
366    Ok(count
367        .zip(names_index)
368        .map(|(count, names_index)| SectionTable {
369            count,
370            names_index,
371            ..sections
372        }))
373}
374
375/// Where an ELF64 little-endian object keeps its section headers.
376struct SectionTable {
377    offset: u64,
378    entry_len: usize,
379    count: usize,
380    names_index: usize,
381}
382
383impl SectionTable {
384    /// Anything but ELF64 little-endian is not an object this pipeline builds.
385    fn parse(header: &[u8]) -> Option<Self> {
386        if header.get(..ELF_IDENT.len())? != ELF_IDENT {
387            return None;
388        }
389        Some(Self {
390            offset: le_bytes(header, E_SHOFF).map(u64::from_le_bytes)?,
391            entry_len: le_bytes(header, E_SHENTSIZE)
392                .map(u16::from_le_bytes)?
393                .into(),
394            count: le_bytes(header, E_SHNUM).map(u16::from_le_bytes)?.into(),
395            names_index: le_bytes(header, E_SHSTRNDX).map(u16::from_le_bytes)?.into(),
396        })
397    }
398
399    fn len(&self) -> Option<usize> {
400        self.entry_len.checked_mul(self.count)
401    }
402
403    fn entry<'a>(&self, table: &'a [u8], index: usize) -> Option<&'a [u8]> {
404        let start = self.entry_len.checked_mul(index)?;
405        table.get(start..start.checked_add(self.entry_len)?)
406    }
407}
408
409/// Reads `len` bytes at `offset`; `None` when the region is implausibly large
410/// or runs past the end of the file.
411fn read_region(
412    file: &mut (impl Read + Seek),
413    offset: u64,
414    len: Option<usize>,
415) -> io::Result<Option<Vec<u8>>> {
416    let Some(len) = len.filter(|len| u64::try_from(*len).is_ok_and(|len| len <= MAX_TABLE_LEN))
417    else {
418        return Ok(None);
419    };
420    file.seek(SeekFrom::Start(offset))?;
421    let mut region = vec![0; len];
422    Ok(fill(file, &mut region)?.then_some(region))
423}
424
425/// `false` when the file ends first: a truncated file is not an object.
426fn fill(file: &mut impl Read, buf: &mut [u8]) -> io::Result<bool> {
427    match file.read_exact(buf) {
428        Ok(()) => Ok(true),
429        Err(err) if err.kind() == ErrorKind::UnexpectedEof => Ok(false),
430        Err(err) => Err(err),
431    }
432}
433
434fn section_name(names: &[u8], offset: u32) -> Option<&[u8]> {
435    names
436        .get(usize::try_from(offset).ok()?..)?
437        .split(|byte| *byte == 0)
438        .next()
439}
440
441fn le_bytes<const N: usize>(bytes: &[u8], at: usize) -> Option<[u8; N]> {
442    bytes.get(at..at.checked_add(N)?)?.try_into().ok()
443}
444
445fn render_html(grcov: &str, lcov_dir: &Path, coverage_dir: &Path) -> Result<()> {
446    // Stale trees must go first: `mv target/doc <existing>` nests the new tree
447    // inside the old one, silently pinning published pages to a snapshot.
448    remove_dir_if_present(coverage_dir)?;
449    run(
450        grcov,
451        &[
452            &lcov_dir.to_string_lossy(),
453            "-s",
454            ".",
455            "-t",
456            "html",
457            "--branch",
458            "--ignore-not-existing",
459            "--ignore",
460            "/*",
461            "--ignore",
462            "*cranelift*",
463            "--ignore",
464            "*lexicon*",
465            "--ignore",
466            "*incremental*",
467            "-o",
468            &coverage_dir.to_string_lossy(),
469        ],
470        &[],
471    )?;
472
473    // grcov exits zero on an empty profile set, so the tree existing is not the
474    // same as the tree having content. This job is the only place that can tell,
475    // and it is the artifact deploy will overlay.
476    let pages = std::fs::read_dir(coverage_dir)
477        .with_context(|| format!("grcov produced no {}", coverage_dir.display()))?
478        .count();
479    if pages == 0 {
480        bail!(
481            "{} is empty — the coverage report has no pages",
482            coverage_dir.display()
483        );
484    }
485    println!(">> coverage report: {pages} entries");
486    Ok(())
487}
488
489#[cfg(test)]
490mod test_fixtures;
491#[cfg(test)]
492mod tests;
493#[cfg(test)]
494mod tool_tests;
xtask/coverage.rs

xtask/
coverage.rs
