Grcov report - fleet_verdict

1

//! Overlapping runs must not overwrite each other's verdicts.

2

//!

3

//! Two releases' migrators can be in flight at once — a retried old Job beside a

4

//! new one. Each run fences its writes on `users.verdict_at`, so a run may only

5

//! rule on a tenant whose last verdict predates it.

6

//!

7

//! The fence deliberately is NOT `quarantined_at`: clearing a quarantine nulls

8

//! that column, which would erase the fence and let an older failure land on a

9

//! tenant a newer run had just verified.

10

#![cfg(feature = "db")]

11

12

use sqlx::Row;

13

use tests_integration::fleet_support::{admin_connect, forget_tenant, register_tenant};

14

use uuid::Uuid;

15

16

6

async fn verdict_state(admin: &sqlx::PgPool, id: Uuid) -> (Option<String>, bool) {

17

6

    let row =

18

6

        sqlx::query("SELECT quarantine_reason, verdict_at IS NOT NULL FROM users WHERE id = $1")

19

6

            .bind(id)

20

6

            .fetch_one(admin)

21

6

            .await

22

6

            .expect("read verdict");

23

6

    (row.get(0), row.get(1))

24

6

25

26

/// Applies the shipped SQL directly, so the test exercises the real statements

27

/// rather than a paraphrase of them.

28

5

async fn quarantine(admin: &sqlx::PgPool, id: Uuid, reason: &str, run: &str) -> u64 {

29

5

    sqlx::query(include_str!("../../server/sql/update/users/quarantine.sql"))

30

5

        .bind(id)

31

5

        .bind(reason)

32

5

        .bind(sqlx::types::chrono::DateTime::parse_from_rfc3339(run).expect("timestamp"))

33

5

        .execute(admin)

34

5

        .await

35

5

        .expect("quarantine")

36

5

        .rows_affected()

37

5

38

39

4

async fn clear(admin: &sqlx::PgPool, id: Uuid, run: &str) -> u64 {

40

4

    sqlx::query(include_str!(

41

4

        "../../server/sql/update/users/clear_quarantine.sql"

42

4

))

43

4

    .bind(id)

44

4

    .bind(sqlx::types::chrono::DateTime::parse_from_rfc3339(run).expect("timestamp"))

45

4

    .execute(admin)

46

4

    .await

47

4

    .expect("clear")

48

4

    .rows_affected()

49

4

50

51

const OLD_RUN: &str = "2020-01-01T00:00:00Z";

52

const NEW_RUN: &str = "2099-01-01T00:00:00Z";

53

54

#[tokio::test]

55

1

async fn an_older_run_cannot_overwrite_a_newer_verdict() -> anyhow::Result<()> {

56

1

    let admin = admin_connect().await;

57

1

    let (id, database_name) = register_tenant(&admin, "fence", false).await;

58

59

    // A newer run verifies the tenant and lifts any quarantine.

60

1

    assert_eq!(clear(&admin, id, NEW_RUN).await, 1);

61

1

    let (reason, has_verdict) = verdict_state(&admin, id).await;

62

1

    assert_eq!(reason, None);

63

1

    assert!(

64

1

        has_verdict,

65

        "a clear records a verdict, it does not erase one"

66

);

67

68

    // An older run, still in flight, now fails on that same tenant. Fencing on

69

    // `quarantined_at` would admit this write, because the clear set it to NULL.

70

1

    assert_eq!(

71

1

        quarantine(&admin, id, "stale failure", OLD_RUN).await,

72

0,

73

        "an older run must not re-quarantine a tenant a newer run just verified"

74

);

75

1

    assert_eq!(verdict_state(&admin, id).await.0, None);

76

77

1

    forget_tenant(&admin, id, &database_name).await;

78

2

    Ok(())

79

1

80

81

#[tokio::test]

82

1

async fn an_older_run_cannot_clear_a_newer_quarantine() -> anyhow::Result<()> {

83

1

    let admin = admin_connect().await;

84

1

    let (id, database_name) = register_tenant(&admin, "fence-inverse", false).await;

85

86

1

    assert_eq!(quarantine(&admin, id, "newer failure", NEW_RUN).await, 1);

87

88

1

    assert_eq!(

89

1

        clear(&admin, id, OLD_RUN).await,

90

0,

91

        "an older success must not release a tenant a newer run has condemned"

92

);

93

1

    assert_eq!(

94

1

        verdict_state(&admin, id).await.0,

95

1

        Some("newer failure".to_owned())

96

);

97

98

1

    forget_tenant(&admin, id, &database_name).await;

99

2

    Ok(())

100

1

101

102

#[tokio::test]

103

1

async fn a_newer_run_always_wins() -> anyhow::Result<()> {

104

1

    let admin = admin_connect().await;

105

1

    let (id, database_name) = register_tenant(&admin, "fence-newer", false).await;

106

107

1

    assert_eq!(quarantine(&admin, id, "old failure", OLD_RUN).await, 1);

108

1

    assert_eq!(

109

1

        clear(&admin, id, NEW_RUN).await,

110

1,

111

        "the fence must not become a deadlock: a newer run still rules"

112

);

113

1

    assert_eq!(verdict_state(&admin, id).await.0, None);

114

115

1

    forget_tenant(&admin, id, &database_name).await;

116

2

    Ok(())

117

1

118

119

#[tokio::test]

120

1

async fn a_late_publication_from_an_older_run_cannot_block_a_newer_one() -> anyhow::Result<()> {

121

    // The realistic overlap, and the one the first version of this fence got

122

    // wrong: BOTH runs start before EITHER publishes. The old run then writes

123

    // last in wall-clock terms.

124

//

125

    // Storing the publication instant while comparing against a run's start

126

    // made the old run stamp a time later than the new run's start, so the new

127

    // run's verdict was refused and the tenant kept the stale one. Both sides

128

    // must be the run's own start.

129

1

    let admin = admin_connect().await;

130

1

    let (id, database_name) = register_tenant(&admin, "fence-late", false).await;

131

132

    const OLDER_START: &str = "2030-01-01T00:00:00Z";

133

    const NEWER_START: &str = "2030-06-01T00:00:00Z";

134

135

    // The older run publishes first, but it started earlier.

136

1

    assert_eq!(

137

1

        quarantine(&admin, id, "stale failure", OLDER_START).await,

138

139

);

140

141

    // The newer run, which started after it, must still be able to rule.

142

1

    assert_eq!(

143

1

        clear(&admin, id, NEWER_START).await,

144

1,

145

        "a newer run must not be blocked by an older run that happened to write first"

146

);

147

1

    assert_eq!(verdict_state(&admin, id).await.0, None);

148

149

    // And the older run, publishing again even later, still cannot overwrite it.

150

1

    assert_eq!(

151

1

        quarantine(&admin, id, "stale again", OLDER_START).await,

152

0,

153

        "writing later in wall-clock time must not promote an older run"

154

);

155

1

    assert_eq!(verdict_state(&admin, id).await.0, None);

156

157

1

    forget_tenant(&admin, id, &database_name).await;

158

2

    Ok(())

159

1