1 
//! One sick tenant degrades itself, not the fleet — and quarantine is not a
2 
//! tombstone.
3 
#![cfg(feature = "db")]
4 

            

        

            

                5
            

            

                
            

            

                
use server::fleet::migrate_fleet;

            

        

            

                6
            

            

                
            

            

                
use tests_integration::fleet_support::{

            

        

            

                7
            

            

                
            

            

                
    accounted_for, admin_connect, admin_id, create_database, forget_tenant, is_fully_migrated,

            

        

            

                8
            

            

                
            

            

                
    quarantine_of, quarantine_reason_in, register_tenant,

            

        

            

                9
            

            

                
            

            

                
};

            

        

            

                10
            

            

                
            

            

                

            

        

            

                11
            

            

                
            

            

                
#[tokio::test]

            

        

            

                12
            

            

                1
            

            

                
async fn an_unreachable_tenant_is_quarantined_and_later_released() -> anyhow::Result<()> {

            

        

            

                13
            

            

                1
            

            

                
    let admin = admin_connect().await;

            

        

            

                14
            

            

                1
            

            

                
    let (healthy, healthy_db) = register_tenant(&admin, "fleet-ok", true).await;

            

        

            

                15
            

            

                1
            

            

                
    let (missing, missing_db) = register_tenant(&admin, "fleet-gone", false).await;

            

        

            

                16
            

            

                
            

            

                

            

        

            

                17
            

            

                1
            

            

                
    let report = migrate_fleet(admin_id()).await?;

            

        

            

                18
            

            

                
            

            

                

            

        

            

                19
            

            

                1
            

            

                
    assert!(

            

        

            

                20
            

            

                1
            

            

                
        quarantine_reason_in(&report, missing).is_some(),

            

        

            

                21
            

            

                
            

            

                
        "the unreachable tenant must be the one quarantined: {report:?}"

            

        

            

                22
            

            

                
            

            

                
    );

            

        

            

                23
            

            

                1
            

            

                
    assert!(

            

        

            

                24
            

            

                1
            

            

                
        quarantine_of(&admin, missing).await.is_some(),

            

        

            

                25
            

            

                
            

            

                
        "the failure must be recorded in the global directory, the only place an \

            

        

            

                26
            

            

                
            

            

                
         unreachable tenant's failure can live"

            

        

            

                27
            

            

                
            

            

                
    );

            

        

            

                28
            

            

                1
            

            

                
    assert!(accounted_for(&report, healthy), "{report:?}");

            

        

            

                29
            

            

                1
            

            

                
    assert!(

            

        

            

                30
            

            

                1
            

            

                
        is_fully_migrated(&healthy_db).await,

            

        

            

                31
            

            

                
            

            

                
        "a healthy tenant must not be held back by a sick one"

            

        

            

                32
            

            

                
            

            

                
    );

            

        

            

                33
            

            

                1
            

            

                
    assert_eq!(quarantine_of(&admin, healthy).await, None);

            

        

            

                34
            

            

                
            

            

                

            

        

            

                35
            

            

                
            

            

                
    // Repair it. A transient outage must not strand a user permanently, so a

            

        

            

                36
            

            

                
            

            

                
    // later run has to retry the tenant and lift the quarantine.

            

        

            

                37
            

            

                1
            

            

                
    create_database(&admin, &missing_db).await;

            

        

            

                38
            

            

                1
            

            

                
    let report = migrate_fleet(admin_id()).await?;

            

        

            

                39
            

            

                
            

            

                

            

        

            

                40
            

            

                1
            

            

                
    assert!(accounted_for(&report, missing), "{report:?}");

            

        

            

                41
            

            

                1
            

            

                
    assert_eq!(

            

        

            

                42
            

            

                1
            

            

                
        quarantine_of(&admin, missing).await,

            

        

            

                43
            

            

                
            

            

                
        None,

            

        

            

                44
            

            

                
            

            

                
        "only a verified success lifts a quarantine, and this was one"

            

        

            

                45
            

            

                
            

            

                
    );

            

        

            

                46
            

            

                1
            

            

                
    assert!(is_fully_migrated(&missing_db).await);

            

        

            

                47
            

            

                
            

            

                

            

        

            

                48
            

            

                1
            

            

                
    forget_tenant(&admin, healthy, &healthy_db).await;

            

        

            

                49
            

            

                1
            

            

                
    forget_tenant(&admin, missing, &missing_db).await;

            

        

            

                50
            

            

                2
            

            

                
    Ok(())

            

        

            

                51
            

            

                1
            

            

                
}