1 
//! Two migrators running at once must not corrupt a tenant or lose it.
2 
//!
3 
//! The per-tenant advisory lock is taken inside that tenant's own database, so a
4 
//! second worker either waits for the claim and then re-runs the (idempotent)
5 
//! migrator, or reports the tenant unresolved. What must never happen is two
6 
//! workers migrating one database concurrently, or both declaring success over a
7 
//! tenant neither finished.
8 
#![cfg(feature = "db")]
9 

            

        

            

                10
            

            

                
            

            

                
use server::fleet::migrate_fleet;

            

        

            

                11
            

            

                
            

            

                
use tests_integration::fleet_support::{

            

        

            

                12
            

            

                
            

            

                
    admin_connect, admin_id, forget_tenant, is_fully_migrated, quarantine_of, register_tenant,

            

        

            

                13
            

            

                
            

            

                
};

            

        

            

                14
            

            

                
            

            

                

            

        

            

                15
            

            

                
            

            

                
#[tokio::test]

            

        

            

                16
            

            

                1
            

            

                
async fn concurrent_migrators_do_not_corrupt_or_abandon_a_tenant() -> anyhow::Result<()> {

            

        

            

                17
            

            

                1
            

            

                
    let admin = admin_connect().await;

            

        

            

                18
            

            

                1
            

            

                
    let (id, database_name) = register_tenant(&admin, "fleet-race", true).await;

            

        

            

                19
            

            

                
            

            

                

            

        

            

                20
            

            

                1
            

            

                
    let (left, right) = tokio::join!(migrate_fleet(admin_id()), migrate_fleet(admin_id()));

            

        

            

                21
            

            

                1
            

            

                
    let (left, right) = (left?, right?);

            

        

            

                22
            

            

                
            

            

                

            

        

            

                23
            

            

                1
            

            

                
    assert!(

            

        

            

                24
            

            

                1
            

            

                
        is_fully_migrated(&database_name).await,

            

        

            

                25
            

            

                
            

            

                
        "whichever worker won, the tenant must end at the current schema"

            

        

            

                26
            

            

                
            

            

                
    );

            

        

            

                27
            

            

                1
            

            

                
    assert_eq!(

            

        

            

                28
            

            

                1
            

            

                
        quarantine_of(&admin, id).await,

            

        

            

                29
            

            

                
            

            

                
        None,

            

        

            

                30
            

            

                
            

            

                
        "losing a race is not a failure and must not quarantine the tenant"

            

        

            

                31
            

            

                
            

            

                
    );

            

        

            

                32
            

            

                
            

            

                

            

        

            

                33
            

            

                
            

            

                
    // Every tenant must be accounted for by at least one of the two runs: an

            

        

            

                34
            

            

                
            

            

                
    // unresolved claim is honest, but both runs skipping it is not.

            

        

            

                35
            

            

                1
            

            

                
    let seen_by = |r: &server::fleet::FleetReport| !r.unresolved.contains(&id);

            

        

            

                36
            

            

                1
            

            

                
    assert!(

            

        

            

                37
            

            

                1
            

            

                
        seen_by(&left) || seen_by(&right),

            

        

            

                38
            

            

                
            

            

                
        "left={left:?} right={right:?}"

            

        

            

                39
            

            

                
            

            

                
    );

            

        

            

                40
            

            

                
            

            

                

            

        

            

                41
            

            

                1
            

            

                
    forget_tenant(&admin, id, &database_name).await;

            

        

            

                42
            

            

                2
            

            

                
    Ok(())

            

        

            

                43
            

            

                1
            

            

                
}