Grcov report - release.rs

1

//! The artifact-producing phases.

2

//!

3

//! Ported from `ci/scripts/release-artifact.sh`. The assertions are the part

4

//! worth keeping: each one exists because something once shipped wrong, and

5

//! several of them are the only thing standing between a green run and an image

6

//! that is missing half its content.

7

8

use std::path::{Path, PathBuf};

9

use std::process::Command;

10

11

use anyhow::{Context, Result, bail};

12

13

use crate::artifact::{Identity, Kind};

14

use crate::heartbeat::Heartbeat;

15

use crate::publish::Store;

16

17

const DOC_DIR: &str = "web/static/doc";

18

19

/// Runs a command, inheriting stdio so its output reaches the job log.

20

fn run(program: &str, args: &[&str], env: &[(&str, &str)]) -> Result<()> {

21

    let mut command = Command::new(program);

22

    command.args(args);

23

    for (key, value) in env {

24

        command.env(key, value);

25

26

    let status = command

27

        .status()

28

        .with_context(|| format!("failed to run {program}"))?;

29

    if !status.success() {

30

        bail!("{program} {} exited with {status}", args.join(" "));

31

32

    Ok(())

33

34

35

/// Instrumented test run plus grcov, producing `web/static/doc/coverage`.

36

pub fn coverage(heartbeat: &Heartbeat, database_url: &str) -> Result<()> {

37

    // LLVM_PROFILE_FILE must be unique per process: %p (pid) and %m (binary id)

38

    // keep the 85 test binaries from overwriting each other's profiles.

39

    let profraw_dir = std::env::current_dir()?.join("target/coverage");

40

    let _ = std::fs::remove_dir_all(&profraw_dir);

41

    std::fs::create_dir_all(&profraw_dir).context("cannot make the profraw dir")?;

42

    let profile_pattern = profraw_dir.join("nomisync-%p-%m.profraw");

43

44

    heartbeat.phase("instrumented test run");

45

    // Watched, unlike every other phase here: this one prints a line per test,

46

    // so silence means something is stuck rather than merely slow. A hang here

47

    // once cost an hour and told us nothing about which test caused it.

48

    crate::watchdog::run_watched(

49

        "cargo",

50

        &["test", "--verbose", "--all-features", "--workspace"],

51

&[

52

            // No -Ccodegen-units=1: LLVM source-based coverage instruments per

53

            // function, so the CGU count never changed what grcov reports — it

54

            // only serialised codegen.

55

            ("RUSTFLAGS", "-Cinstrument-coverage"),

56

            ("CARGO_INCREMENTAL", "0"),

57

            ("LLVM_PROFILE_FILE", &profile_pattern.to_string_lossy()),

58

            ("DATABASE_URL", database_url),

59

],

60

)?;

61

62

    // A coverage run that produced no profiles yields an empty-but-valid grcov

63

    // report: green CI, blank coverage page. Assert instead.

64

    let profraw_count = std::fs::read_dir(&profraw_dir)

65

        .context("cannot read the profraw dir")?

66

        .filter_map(Result::ok)

67

        .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "profraw"))

68

        .count();

69

    if profraw_count == 0 {

70

        bail!("no .profraw produced — coverage instrumentation did not run");

71

72

    println!(">> {profraw_count} profraw files");

73

74

    heartbeat.phase("grcov");

75

    // Stale trees must go first: `mv target/doc <existing>` nests the new tree

76

    // inside the old one, silently pinning published pages to a snapshot.

77

    let coverage_dir = PathBuf::from(DOC_DIR).join("coverage");

78

    let _ = std::fs::remove_dir_all(&coverage_dir);

79

    run(

80

        "grcov",

81

&[

82

            ".",

83

            "-s",

84

            ".",

85

            "--binary-path",

86

            "./target/debug/",

87

            "-t",

88

            "html",

89

            "--branch",

90

            "--ignore-not-existing",

91

            "--ignore",

92

            "/*",

93

            "--ignore",

94

            "*cranelift*",

95

            "--ignore",

96

            "*lexicon*",

97

            "--ignore",

98

            "*incremental*",

99

            "-o",

100

            &coverage_dir.to_string_lossy(),

101

],

102

        &[],

103

)?;

104

105

    // grcov exits zero on an empty profile set, so the tree existing is not the

106

    // same as the tree having content. This job is the only place that can tell,

107

    // and it is the artifact deploy will overlay.

108

    let pages = std::fs::read_dir(&coverage_dir)

109

        .with_context(|| format!("grcov produced no {}", coverage_dir.display()))?

110

        .count();

111

    if pages == 0 {

112

        bail!(

113

            "{} is empty — the coverage report has no pages",

114

            coverage_dir.display()

115

);

116

117

    println!(">> coverage report: {pages} entries");

118

    Ok(())

119

120

121

/// rustdoc, the org exports, and the static musl binaries.

122

pub fn build(heartbeat: &Heartbeat, database_url: &str) -> Result<()> {

123

    let doc_dir = PathBuf::from(DOC_DIR);

124

125

    heartbeat.phase("rustdoc");

126

    for stale in ["rustdoc", "scripts", "nomiscript"] {

127

        let _ = std::fs::remove_dir_all(doc_dir.join(stale));

128

129

    let _ = std::fs::remove_file(doc_dir.join("disdoc.html"));

130

    // cargo doc recompiles the sqlx macros, so this phase needs a live schema

131

    // even though it produces no test results.

132

    run(

133

        "cargo",

134

        &["doc", "--workspace", "--all-features", "--no-deps"],

135

        &[("DATABASE_URL", database_url)],

136

)?;

137

    std::fs::rename("target/doc", doc_dir.join("rustdoc"))

138

        .context("cannot move the rustdoc tree into the static dir")?;

139

140

    heartbeat.phase("org exports");

141

    std::fs::create_dir_all(doc_dir.join("scripts")).ok();

142

    std::fs::create_dir_all(doc_dir.join("nomiscript")).ok();

143

    export_org(Path::new("doc/disdoc.org"))?;

144

    std::fs::rename("doc/disdoc.html", doc_dir.join("disdoc.html"))

145

        .context("cannot move disdoc.html")?;

146

    export_dir(Path::new("doc/scripts"), &doc_dir.join("scripts"))?;

147

    export_dir(Path::new("doc/nomiscript"), &doc_dir.join("nomiscript"))?;

148

149

    heartbeat.phase("musl release build");

150

    // CXX stays because -sys crates that build C++ need it, and the image has no

151

    // x86_64-linux-musl-g++ — only the gcc driver.

152

    run(

153

        "cargo",

154

&[

155

            "build",

156

            "--release",

157

            "--target",

158

            "x86_64-unknown-linux-musl",

159

            "-p",

160

            "web",

161

            "-p",

162

            "sshd",

163

],

164

&[

165

            ("SQLX_OFFLINE", "true"),

166

            ("CC_x86_64_unknown_linux_musl", "x86_64-linux-musl-gcc"),

167

            ("CXX_x86_64_unknown_linux_musl", "x86_64-linux-musl-gcc"),

168

],

169

)?;

170

171

    std::fs::copy(

172

        "target/x86_64-unknown-linux-musl/release/web",

173

        "release-web",

174

175

    .context("cannot stage release-web")?;

176

    std::fs::copy(

177

        "target/x86_64-unknown-linux-musl/release/nomisync-sshd",

178

        "release-sshd",

179

180

    .context("cannot stage release-sshd")?;

181

182

    verify_binaries()?;

183

    stage_licences()?;

184

    Ok(())

185

186

187

/// One emacs invocation PER FILE.

188

///

189

/// `emacs --batch a.org b.org --eval (org-html-export-to-html)` runs the eval

190

/// once, against the last visited buffer, so a glob silently publishes a single

191

/// page — the retired Drone pipeline shipped 1 of 4 script pages exactly that way.

192

fn export_org(org: &Path) -> Result<()> {

193

    run(

194

        "emacs",

195

&[

196

            "-q",

197

            "--batch",

198

            &org.to_string_lossy(),

199

            "-l",

200

            "ox-html",

201

            "--eval",

202

            "(progn (org-babel-do-load-languages (quote org-babel-load-languages) (quote ((emacs-lisp . t) (shell . t)))) (setq org-confirm-babel-evaluate nil) (setq org-html-validation-link nil) (org-html-export-to-html))",

203

            "-f",

204

            "save-buffer",

205

            "-f",

206

            "kill-emacs",

207

],

208

        &[],

209

210

211

212

fn export_dir(src: &Path, dest: &Path) -> Result<()> {

213

    let orgs: Vec<PathBuf> = std::fs::read_dir(src)

214

        .with_context(|| format!("cannot read {}", src.display()))?

215

        .filter_map(Result::ok)

216

        .map(|entry| entry.path())

217

        .filter(|path| path.extension().is_some_and(|ext| ext == "org"))

218

        .collect();

219

    if orgs.is_empty() {

220

        bail!("no org files in {}", src.display());

221

222

223

    for org in &orgs {

224

        export_org(org)?;

225

226

227

    // Count what was produced, not what was attempted.

228

    let produced: Vec<PathBuf> = std::fs::read_dir(src)?

229

        .filter_map(Result::ok)

230

        .map(|entry| entry.path())

231

        .filter(|path| path.extension().is_some_and(|ext| ext == "html"))

232

        .collect();

233

    if produced.len() != orgs.len() {

234

        bail!(

235

            "{}: exported {} of {} pages",

236

            src.display(),

237

            produced.len(),

238

            orgs.len()

239

);

240

241

    for page in produced {

242

        let target = dest.join(page.file_name().context("html page has no file name")?);

243

        std::fs::rename(&page, &target)

244

            .with_context(|| format!("cannot move {}", page.display()))?;

245

246

    println!(">> exported {} pages from {}", orgs.len(), src.display());

247

    Ok(())

248

249

250

/// The assertions that prove the shipped binaries are what we think they are.

251

fn verify_binaries() -> Result<()> {

252

    for binary in ["release-web", "release-sshd"] {

253

        let bytes = std::fs::read(binary).with_context(|| format!("cannot read {binary}"))?;

254

255

        // Coverage instrumentation in a shipped binary is invisible until

256

        // production is slow and writing .profraw files.

257

        if contains(&bytes, b"__llvm_profile") {

258

            bail!(

259

                "{binary} contains coverage instrumentation — RUSTFLAGS leaked out of the coverage phase"

260

);

261

262

263

        // plotters prefers its `ttf` backend when both are enabled, and that

264

        // backend dlopens fontconfig — impossible in a static musl binary, so

265

        // chart rendering PANICS rather than degrading. The builder still

266

        // carries the fontconfig packages, which would let such a regression

267

        // build cleanly and only fail inside the scratch image.

268

        if binary == "release-web" && contains(&bytes, b"fontconfig") {

269

            bail!(

270

                "release-web references fontconfig — plotters' ttf backend is back; it cannot work in the scratch image"

271

);

272

273

274

275

    // The generated trees are gitignored, so their absence here means a phase

276

    // above failed quietly. The deploy job asserts the same on the unpacked side.

277

//

278

    // NOT coverage: that is the coverage job's output, published as its own

279

    // artifact and overlaid at deploy time. Asserting it here is what made the

280

    // first xtask release run fail — the phase had moved, the assertion had not.

281

    for required in ["rustdoc", "scripts", "nomiscript", "disdoc.html"] {

282

        let path = PathBuf::from(DOC_DIR).join(required);

283

        if !path.exists() {

284

            bail!("missing {}", path.display());

285

286

287

    Ok(())

288

289

290

fn contains(haystack: &[u8], needle: &[u8]) -> bool {

291

    haystack

292

        .windows(needle.len())

293

        .any(|window| window == needle)

294

295

296

/// release-web embeds DejaVuSans via include_bytes!, and that font's licence

297

/// requires its notice to travel with copies of the font software. web/static is

298

/// copied wholesale into the runtime image, so putting it here makes it ship

299

/// with both the tarball and the image, and be servable.

300

fn stage_licences() -> Result<()> {

301

    let dir = PathBuf::from("web/static/licenses");

302

    std::fs::create_dir_all(&dir).context("cannot make the licenses dir")?;

303

    let target = dir.join("DejaVuSans-LICENSE.txt");

304

    std::fs::copy("plotting/fonts/LICENSE", &target).context("cannot copy the font licence")?;

305

    if std::fs::metadata(&target)?.len() == 0 {

306

        bail!("font licence missing from artifact");

307

308

    Ok(())

309

310

311

/// Tars the payload for a kind and, when this run holds the write key, publishes it.

312

pub fn package(kind: Kind, heartbeat: &Heartbeat, with_artifacts: bool) -> Result<()> {

313

    heartbeat.phase("packaging");

314

315

    let tarball = PathBuf::from(format!("nomisync-{kind}.tar"));

316

    let members: &[&str] = match kind {

317

        // Only the coverage tree, so the two payloads are disjoint by rule and

318

        // the assembler can reject any overlap rather than guess.

319

        Kind::Coverage => &["web/static/doc/coverage"],

320

        Kind::Release => &["release-web", "release-sshd", "web/static"],

321

};

322

    let mut args = vec!["-czf", &tarball.to_str().context("tarball path")?];

323

    args.extend_from_slice(members);

324

    run("tar", &args, &[])?;

325

326

    if !with_artifacts {

327

        println!(">> WITH_ARTIFACTS unset — built and verified, not uploading");

328

        return Ok(());

329

330

331

    let identity = Identity::from_env()?;

332

    let store = Store::from_env()?;

333

    let meta = store.publish(kind, &tarball, &identity)?;

334

    println!(

335

        ">> published {} ({} bytes) for {} run {}",

336

        meta.object_key(),

337

        meta.bytes,

338

        identity.commit,

339

        identity.run_id

340

);

341

    Ok(())

342