diff --git a/simulator/equal.toml b/simulator/equal.toml new file mode 100644 index 0000000..3212071 --- /dev/null +++ b/simulator/equal.toml @@ -0,0 +1,23 @@ +food_amount = 100 +view_dist = 200 +red_count = 10 +blu_count = 10 + +[weapon] +kills_user = false +type = "ball" +radius = 200 + +[red_policy] +aggressive = false +food = [1,0] +kin = [-0.3,0.02] +nonkin = [-0.3,-0.02] +none= [0.3, 0.01] + +[blu_policy] +aggressive = false +food = [1,0] +kin = [-0.3,0.02] +nonkin = [-0.3,-0.02] +none= [0.3, 0.01] diff --git a/simulator/src/main.rs b/simulator/src/main.rs index f6e7797..614e616 100644 --- a/simulator/src/main.rs +++ b/simulator/src/main.rs @@ -661,6 +661,85 @@ fn load_config(path: &std::path::Path) -> SimConfig { toml::from_str(&text).unwrap() } +/// Online mean/variance accumulator (Welford's algorithm), used to report how +/// much headless outcomes vary from seed to seed rather than only their mean. +#[derive(Debug, Clone, Copy, Default)] +struct Variance { + n: u64, + mean: f64, + m2: f64, +} + +impl Variance { + fn push(&mut self, x: f64) { + self.n += 1; + let delta = x - self.mean; + self.mean += delta / self.n as f64; + self.m2 += delta * (x - self.mean); + } + + /// Unbiased sample variance (`n - 1` denominator); `None` for `n < 2`. + fn sample_variance(&self) -> Option { + (self.n > 1).then(|| self.m2 / (self.n - 1) as f64) + } + + fn std_dev(&self) -> Option { + self.sample_variance().map(f64::sqrt) + } + + fn std_err(&self) -> Option { + self.std_dev().map(|s| s / (self.n as f64).sqrt()) + } + + /// Confidence interval for the mean, using `z` as the critical value of the + /// standard normal distribution (e.g. 1.96 for ~95%). Normal is a good + /// approximation here because the harness normally runs many samples. + fn confidence_interval(&self, z: f64) -> Option<(f64, f64)> { + self.std_err() + .map(|se| (self.mean - z * se, self.mean + z * se)) + } + + fn describe(&self) -> String { + match (self.sample_variance(), self.std_dev(), self.confidence_interval(1.96)) { + (Some(var), Some(sd), Some((lo, hi))) => format!( + "{:.4} (sd {sd:.4}, var {var:.4}, 95% CI [{lo:.4}, {hi:.4}])", + self.mean + ), + _ => format!("{:.4} (n={}, no variance estimate)", self.mean, self.n), + } + } +} + +/// Summarise per-seed headless runs in terms of each team's proportion of the +/// initial food, including the variance of those proportions and of the RED-BLU +/// difference (which is what a fair match should centre on zero). +fn print_run_stats(results: &[(i32, i32)], food_amount: usize) { + if food_amount == 0 { + println!("no food configured (food_amount = 0); nothing to measure"); + return; + } + + let mut red = Variance::default(); + let mut blu = Variance::default(); + let mut diff = Variance::default(); + for &(r, b) in results { + let r_share = r as f64 / food_amount as f64; + let b_share = b as f64 / food_amount as f64; + red.push(r_share); + blu.push(b_share); + diff.push(r_share - b_share); + } + + let winner = if red.mean > blu.mean { "red" } else { "blue" }; + println!( + "{winner} wins on average over {} runs! (share of initial {food_amount} food)", + results.len().max(1) + ); + println!(" RED share: {}", red.describe()); + println!(" BLU share: {}", blu.describe()); + println!(" RED-BLU share: {}", diff.describe()); +} + fn print_average(red: i32, blu: i32, iters: u64) { let iters = iters.max(1) as f32; let red = red as f32 / iters; @@ -677,11 +756,11 @@ fn main() { if cli.no_gui { let config = load_config(&cli.config); let iters = cli.run_count.unwrap_or(1); - let (red, blu) = (0..iters) + let results: Vec<(i32, i32)> = (0..iters) .into_par_iter() .map(|s| run_headless_simulation(s, &config)) - .reduce(|| (0, 0), |a, b| (a.0 + b.0, a.1 + b.1)); - print_average(red, blu, iters); + .collect(); + print_run_stats(&results, config.food_amount); } else { macroquad::Window::new("Adversarial Foraging Simulator", gui_main(cli)); } @@ -712,6 +791,10 @@ mod tests { toml::from_str(include_str!("../conf.toml")).unwrap() } + fn equal_config() -> SimConfig { + toml::from_str(include_str!("../equal.toml")).unwrap() + } + /// The analytic replacements must agree with parry bit-for-bit, both on /// whether a contact exists and on the reported `point1.distance(origin)`. /// This is the load-bearing test behind the whole speedup: the hot path no @@ -789,6 +872,32 @@ mod tests { assert_eq!(blu, 9088, "blu total over 200 seeds"); } + /// `equal.toml` gives both teams identical policies and mirrored starting + /// geometry, so both teams should collect the same proportion of the + /// initial food. Verify that zero lies inside the 95% confidence interval + /// of the RED-BLU proportion difference. + #[test] + fn equal_config_is_balanced() { + let config = equal_config(); + let mut diff = Variance::default(); + for seed in 0..200u64 { + let (red, blu) = run_headless_simulation(seed, &config); + + let r_share = red as f64 / config.food_amount as f64; + let b_share = blu as f64 / config.food_amount as f64; + diff.push(r_share - b_share); + } + + let (lo, hi) = diff + .confidence_interval(1.96) + .expect("need at least two samples for a confidence interval"); + assert!( + lo <= 0.0 && 0.0 <= hi, + "RED-BLU food share mean {:.4} with 95% CI [{lo:.4}, {hi:.4}] excludes zero", + diff.mean, + ); + } + /// Same seed twice must produce the same result (no hidden global state). #[test] fn deterministic() {