Implement stratified and by-order sampling

docs/options.md

@@ -95,8 +95,6 @@ Random sample methods are also supported, but also not recommended:
 
 The use of a random sample method requires *Normalise ballots* to be enabled, and will usually be used with a *Quota criterion* set to *>=*.
 
-In both random sample methods, the subset is selected using the deterministic method used in [Cambridge, Massachusetts](https://web.archive.org/web/20081118104049/http://www.fairvote.org/media/1993countmanual.pdf) (derived from Article IX of the former 1938 Cincinnati *Code of Ordinances*). This depends on the order of ballot papers in the BLT file, and is independent of the *Random seed* option.
-
 ### Papers to examine in surplus transfer (--t ransferable-only)
 
 * *Include non-transferable papers* (default): When this option is selected, all ballot papers of the transferring candidate are examined. Non-transferable papers are always exhausted at the relevant surplus fractions.
@@ -119,6 +117,16 @@ When *Surplus method* is set to *Meek method*, this option controls how candidat
 * When NZ-style exclusion is disabled (default), the excluded candidate's keep value is immediately reduced to 0. This is the method specified in the 1987 and 2006 Meek rules.
 * When NZ-style exclusion is enabled, all elected candidates' keep values are first updated by one further iteration; only then is the excluded candidate's keep value reduced to 0. This is the method specified in the New Zealand *Local Electoral Regulations 2001*.
 
+### (Sample) Sample method (--sample)
+
+When *Surplus method* is set to a random sample method, this option controls which subset of ballot papers is selected for transfer during surplus distributions:
+
+* *Stratified (then by order)*: The candidate's ballot papers are first stratified into subparcels according to next available preference. From each subparcel, the subset transferred comprises the ballot papers most recently received by the candidate.
+* *By order*: The subset transferred comprises the ballot papers most recently received by the candidate.
+* *Every n-th ballot*: The subset is selected using the deterministic method used in [Cambridge, Massachusetts](https://web.archive.org/web/20081118104049/http://www.fairvote.org/media/1993countmanual.pdf) (derived from Article IX of the former 1938 Cincinnati *Code of Ordinances*).
+
+In any case, the subset selected depends on the order of ballot papers in the BLT file, and is independent of the *Random seed* option.
+
 ### (Sample) Transfer ballot-by-ballot (--sample-per-ballot)
 
 When *Surplus method* is set to a random sample method, this option controls when candidates are declared elected:

html/index.html

@@ -141,9 +141,17 @@
 						</label>
 					</div>
 					<div>
+						<label style="margin-right:1em;">
+							<span class="pill-grey" title="This option has effect only if “Method” is set to a random sample method">Sample</span>
+							Sample method:
+							<select id="selSample">
+								<option value="stratified" selected>Stratified (then by order)</option>
+								<option value="by_order">By order</option>
+								<option value="nth_ballot">Every n-th ballot</option>
+							</select>
+						</label>
 						<label>
 							<input type="checkbox" id="chkSamplePerBallot">
-							<span class="pill-grey" title="This option has effect only if “Method” is set to a sample-based method">Sample</span>
 							Per-ballot transfers
 						</label>
 					</div>

html/index.js

@@ -149,6 +149,7 @@ async function clickCount() {
 		document.getElementById('selPapers').value == 'transferable',
 		document.getElementById('selExclusion').value,
 		document.getElementById('chkMeekNZExclusion').checked,
+		document.getElementById('selSample').value,
 		document.getElementById('chkSamplePerBallot').checked,
 		document.getElementById('chkBulkElection').checked,
 		document.getElementById('chkBulkExclusion').checked,
@@ -566,6 +567,7 @@ function changePreset() {
 		document.getElementById('chkBulkElection').checked = true;
 		document.getElementById('chkBulkExclusion').checked = false;
 		document.getElementById('chkDeferSurpluses').checked = false;
+		document.getElementById('selSample').value = 'nth_ballot';
 		document.getElementById('chkSamplePerBallot').checked = true;
 		document.getElementById('txtMinThreshold').value = '49';
 		document.getElementById('selNumbers').value = 'rational';

src/main.rs

@@ -139,6 +139,10 @@ struct STV {
 	#[clap(help_heading=Some("STV VARIANTS"), long)]
 	meek_nz_exclusion: bool,
 	
+	/// (Cincinnati/Hare) Method of drawing a sample
+	#[clap(help_heading=Some("STV VARIANTS"), long, possible_values=&["stratified", "by_order", "nth_ballot"], default_value="stratified", value_name="method")]
+	sample: String,
+	
 	/// (Cincinnati/Hare) Sample-based methods: Check for candidate election after each individual ballot paper transfer
 	#[clap(help_heading=Some("STV VARIANTS"), long)]
 	sample_per_ballot: bool,
@@ -278,6 +282,7 @@ where
 		cmd_opts.transferable_only,
 		cmd_opts.exclusion.into(),
 		cmd_opts.meek_nz_exclusion,
+		cmd_opts.sample.into(),
 		cmd_opts.sample_per_ballot,
 		!cmd_opts.no_early_bulk_elect,
 		cmd_opts.bulk_exclude,

src/stv/gregory.rs

@@ -115,7 +115,9 @@ where
 	return Ok(false);
 }
 
-/// Return the denominator of the transfer value
+/// Return the denominator of the surplus fraction
+///
+/// Returns `None` if the value of transferable votes <= surplus (i.e. all transferable votes are transferred at values received)
 fn calculate_surplus_denom<N: Number>(surplus: &N, result: &NextPreferencesResult<N>, transferable_votes: &N, weighted: bool, transferable_only: bool) -> Option<N>
 where
 	for<'r> &'r N: ops::Sub<&'r N, Output=N>
@@ -295,10 +297,11 @@ where
 		None => {
 			surplus_fraction = None;
 			
-			if opts.transferable_only {
+			// This can only happen if --transferable-only
-				state.logger.log_literal(format!("Transferring {:.0} transferable ballots, totalling {:.dps$} transferable votes, at values received.", &result.total_ballots - &result.exhausted.num_ballots, transferable_votes, dps=opts.pp_decimals));
+			if &result.total_ballots - &result.exhausted.num_ballots == N::one() {
+				state.logger.log_literal(format!("Transferring 1 transferable ballot, totalling {:.dps$} transferable votes, at values received.", transferable_votes, dps=opts.pp_decimals));
 			} else {
-				state.logger.log_literal(format!("Transferring {:.0} ballots, totalling {:.dps$} votes, at values received.", result.total_ballots, result.total_votes, dps=opts.pp_decimals));
+				state.logger.log_literal(format!("Transferring {:.0} transferable ballots, totalling {:.dps$} transferable votes, at values received.", &result.total_ballots - &result.exhausted.num_ballots, transferable_votes, dps=opts.pp_decimals));
 			}
 		}
 	}

src/stv/mod.rs

@@ -21,7 +21,7 @@
 pub mod gregory;
 /// Meek method of surplus distributions, etc.
 pub mod meek;
-/// Random subset methods of surplus distributions
+/// Random sample methods of surplus distributions
 pub mod sample;
 
 /// WebAssembly wrappers
@@ -110,6 +110,10 @@ pub struct STVOptions {
 	#[builder(default="false")]
 	pub meek_nz_exclusion: bool,
 	
+	/// (Cincinnati/Hare) Method of drawing a sample
+	#[builder(default="SampleMethod::Stratified")]
+	pub sample: SampleMethod,
+	
 	/// (Cincinnati/Hare) Sample-based methods: Check for candidate election after each individual ballot paper transfer
 	#[builder(default="false")]
 	pub sample_per_ballot: bool,
@@ -182,6 +186,7 @@ impl STVOptions {
 			if self.exclusion != ExclusionMethod::SingleStage { flags.push(self.exclusion.describe()); }
 		}
 		if self.surplus == SurplusMethod::Meek && self.meek_nz_exclusion { flags.push("--meek-nz-exclusion".to_string()); }
+		if (self.surplus == SurplusMethod::Cincinnati || self.surplus == SurplusMethod::Hare) && self.sample != SampleMethod::Stratified { flags.push(self.sample.describe()); }
 		if (self.surplus == SurplusMethod::Cincinnati || self.surplus == SurplusMethod::Hare) && self.sample_per_ballot { flags.push("--sample-per-ballot".to_string()); }
 		if !self.early_bulk_elect { flags.push("--no-early-bulk-elect".to_string()); }
 		if self.bulk_exclude { flags.push("--bulk-exclude".to_string()); }
@@ -207,6 +212,7 @@ impl STVOptions {
 		if self.surplus == SurplusMethod::Cincinnati || self.surplus == SurplusMethod::Hare {
 			if self.round_quota != Some(0) { return Err(STVError::InvalidOptions("--surplus cincinnati and --surplus hare require --round-quota 0")); }
 			if !self.normalise_ballots { return Err(STVError::InvalidOptions("--surplus cincinnati and --surplus hare require --normalise-ballots")); }
+			if self.sample == SampleMethod::Stratified && self.sample_per_ballot { return Err(STVError::InvalidOptions("--sample stratified is incompatible with --sample-per-ballot")); }
 		}
 		if self.min_threshold != "0" && self.defer_surpluses { return Err(STVError::InvalidOptions("--min-threshold is incompatible with --defer-surpluses")); } // TODO: Permit this
 		return Ok(());
@@ -474,6 +480,41 @@ impl<S: AsRef<str>> From<S> for ExclusionMethod {
 	}
 }
 
+/// Enum of options for [STVOptions::sample]
+#[wasm_bindgen]
+#[derive(Clone, Copy)]
+#[derive(PartialEq)]
+pub enum SampleMethod {
+	/// Stratify the ballots into parcels according to next available preference and transfer the last ballots from each parcel
+	Stratified,
+	/// Transfer the last ballots
+	ByOrder,
+	/// Transfer every n-th ballot, Cincinnati style
+	NthBallot,
+}
+
+impl SampleMethod {
+	/// Convert to CLI argument representation
+	fn describe(self) -> String {
+		match self {
+			SampleMethod::Stratified => "--sample stratified",
+			SampleMethod::ByOrder => "--sample by_order",
+			SampleMethod::NthBallot => "--sample nth_ballot",
+		}.to_string()
+	}
+}
+
+impl<S: AsRef<str>> From<S> for SampleMethod {
+	fn from(s: S) -> Self {
+		match s.as_ref() {
+			"stratified" => SampleMethod::Stratified,
+			"by_order" => SampleMethod::ByOrder,
+			"nth_ballot" => SampleMethod::NthBallot,
+			_ => panic!("Invalid --sample-method"),
+		}
+	}
+}
+
 /// Enum of options for [STVOptions::constraint_mode]
 #[derive(Clone, Copy)]
 #[derive(PartialEq)]
@@ -1257,10 +1298,14 @@ where
 {
 	let mut excluded_candidates: Vec<&Candidate> = Vec::new();
 	
-	// Exclude candidates below min threshold
 	if state.num_excluded == 0 {
+		if opts.bulk_exclude && opts.min_threshold == "0" {
+			// Proceed directly to bulk exclusion, as candidates with 0 votes will necessarily be included
+		} else {
+			// Exclude candidates below min threshold
 			excluded_candidates = hopefuls_below_threshold(state, opts);
 		}
+	}
 	
 	// Attempt a bulk exclusion
 	if excluded_candidates.is_empty() && opts.bulk_exclude {

src/stv/sample.rs

@@ -18,12 +18,31 @@
 use crate::constraints;
 use crate::election::{Candidate, CandidateState, CountState, Parcel, Vote};
 use crate::numbers::Number;
-use crate::stv::{STVOptions, SurplusMethod};
+use crate::stv::{STVOptions, SampleMethod, SurplusMethod};
 
+use std::cmp::max;
 use std::collections::HashMap;
 use std::ops;
 
-use super::STVError;
+use super::{NextPreferencesResult, STVError};
+
+/// Return the denominator of the surplus fraction
+///
+/// Returns `None` if transferable ballots <= surplus (i.e. all transferable ballots are transferred at full value)
+fn calculate_surplus_denom<N: Number>(surplus: &N, result: &NextPreferencesResult<N>, transferable_ballots: &N, transferable_only: bool) -> Option<N>
+where
+	for<'r> &'r N: ops::Sub<&'r N, Output=N>
+{
+	if transferable_only {
+		if transferable_ballots > surplus {
+			return Some(transferable_ballots.clone());
+		} else {
+			return None;
+		}
+	} else {
+		return Some(result.total_ballots.clone());
+	}
+}
 
 /// Distribute the surplus of a given candidate according to the random subset method, based on [STVOptions::surplus]
 pub fn distribute_surplus<N: Number>(state: &mut CountState<N>, opts: &STVOptions, elected_candidate: &Candidate) -> Result<(), STVError>
@@ -39,7 +58,7 @@ where
 	let count_card = state.candidates.get_mut(elected_candidate).unwrap();
 	let surplus = &count_card.votes - state.quota.as_ref().unwrap();
 	
-	let votes;
+	let mut votes;
 	match opts.surplus {
 		SurplusMethod::Cincinnati => {
 			// Inclusive
@@ -53,6 +72,147 @@ where
 		_ => unreachable!()
 	}
 	
+	match opts.sample {
+		SampleMethod::Stratified => {
+			// Stratified by next available preference
+			// FIXME: This is untested
+			
+			let result = super::next_preferences(state, votes);
+			
+			let transferable_ballots = &result.total_ballots - &result.exhausted.num_ballots;
+			let surplus_denom = calculate_surplus_denom(&surplus, &result, &transferable_ballots, opts.transferable_only);
+			let mut surplus_fraction;
+			match surplus_denom {
+				Some(v) => {
+					surplus_fraction = Some(surplus.clone() / v);
+					
+					// Round down if requested
+					if let Some(dps) = opts.round_surplus_fractions {
+						surplus_fraction.as_mut().unwrap().floor_mut(dps);
+					}
+					
+					if opts.transferable_only {
+						if &result.total_ballots - &result.exhausted.num_ballots == N::one() {
+							state.logger.log_literal(format!("Examining 1 transferable ballot, with surplus fraction {:.dps2$}.", surplus_fraction.as_ref().unwrap(), dps2=max(opts.pp_decimals, 2)));
+						} else {
+							state.logger.log_literal(format!("Examining {:.0} transferable ballots, with surplus fraction {:.dps2$}.", &result.total_ballots - &result.exhausted.num_ballots, surplus_fraction.as_ref().unwrap(), dps2=max(opts.pp_decimals, 2)));
+						}
+					} else {
+						if result.total_ballots == N::one() {
+							state.logger.log_literal(format!("Examining 1 ballot, with surplus fraction {:.dps2$}.", surplus_fraction.as_ref().unwrap(), dps2=max(opts.pp_decimals, 2)));
+						} else {
+							state.logger.log_literal(format!("Examining {:.0} ballots, with surplus fraction {:.dps2$}.", result.total_ballots, surplus_fraction.as_ref().unwrap(), dps2=max(opts.pp_decimals, 2)));
+						}
+					}
+				}
+				None => {
+					surplus_fraction = None;
+					
+					// This can only happen if --transferable-only
+					if result.total_ballots == N::one() {
+						state.logger.log_literal("Transferring 1 ballot at full value.".to_string());
+					} else {
+						state.logger.log_literal(format!("Transferring {:.0} ballots at full value.", result.total_ballots));
+					}
+				}
+			}
+			
+			let mut checksum = N::new();
+			
+			for (candidate, entry) in result.candidates.into_iter() {
+				// Credit transferred votes
+				let mut candidate_transfers;
+				match surplus_fraction {
+					Some(ref f) => {
+						candidate_transfers = entry.num_ballots * f;
+						candidate_transfers.floor_mut(0);
+					}
+					None => {
+						// All ballots transferred
+						candidate_transfers = entry.num_ballots.clone();
+					}
+				}
+				let candidate_transfers_usize: usize = format!("{:.0}", candidate_transfers).parse().expect("Transfers overflow usize");
+				
+				let count_card = state.candidates.get_mut(candidate).unwrap();
+				count_card.transfer(&candidate_transfers);
+				checksum += candidate_transfers;
+				
+				let parcel = Parcel {
+					votes: entry.votes.into_iter().rev().take(candidate_transfers_usize).rev().collect(),
+					source_order: state.num_elected + state.num_excluded,
+				};
+				
+				count_card.parcels.push(parcel);
+			}
+			
+			// Credit exhausted votes
+			let mut exhausted_transfers;
+			if opts.transferable_only {
+				if transferable_ballots > surplus {
+					// No ballots exhaust
+					exhausted_transfers = N::new();
+				} else {
+					exhausted_transfers = &surplus - &transferable_ballots;
+					
+					if let Some(dps) = opts.round_votes {
+						exhausted_transfers.floor_mut(dps);
+					}
+				}
+			} else {
+				exhausted_transfers = result.exhausted.num_ballots * surplus_fraction.as_ref().unwrap();
+				exhausted_transfers.floor_mut(0);
+			}
+			let exhausted_transfers_usize: usize = format!("{:.0}", exhausted_transfers).parse().expect("Transfers overflow usize");
+			
+			state.exhausted.transfer(&exhausted_transfers);
+			checksum += exhausted_transfers;
+			
+			// Transfer exhausted votes
+			let parcel = Parcel {
+				votes: result.exhausted.votes.into_iter().rev().take(exhausted_transfers_usize).rev().collect(),
+				source_order: state.num_elected + state.num_excluded,
+			};
+			state.exhausted.parcels.push(parcel);
+			
+			// Finalise candidate votes
+			let count_card = state.candidates.get_mut(elected_candidate).unwrap();
+			count_card.transfers = -&surplus;
+			count_card.votes.assign(state.quota.as_ref().unwrap());
+			checksum -= surplus;
+			
+			count_card.parcels.clear(); // Mark surpluses as done
+			
+			// Update loss by fraction
+			state.loss_fraction.transfer(&-checksum);
+		}
+		SampleMethod::ByOrder => {
+			// Ballots by order
+			// FIXME: This is untested
+			
+			state.logger.log_literal(format!("Examining {:.0} ballots.", votes.len())); // votes.len() is total ballots as --normalise-ballots is required
+			
+			// Transfer candidate votes
+			while &state.candidates[elected_candidate].votes > state.quota.as_ref().unwrap() {
+				match votes.pop() {
+					Some(vote) => {
+						// Transfer to next preference
+						transfer_ballot(state, opts, elected_candidate, vote)?;
+					}
+					None => {
+						// We have run out of ballot papers
+						// Remaining ballot papers exhaust
+						
+						let surplus = &state.candidates[elected_candidate].votes - state.quota.as_ref().unwrap();
+						state.exhausted.transfer(&surplus);
+						state.candidates.get_mut(elected_candidate).unwrap().transfer(&-surplus);
+					}
+				}
+			}
+		}
+		SampleMethod::NthBallot => {
+			// Every nth-ballot (Cincinnati-style)
+			
 			// Calculate skip value
 			let total_ballots = votes.len();
 			let mut skip_fraction = N::from(total_ballots) / &surplus;
@@ -72,9 +232,37 @@ where
 			let mut index = skip_value - 1; // Subtract 1 as votes are 0-indexed
 			
 			while &state.candidates[elected_candidate].votes > state.quota.as_ref().unwrap() {
-		let mut vote = numbered_votes.remove(&index).unwrap();
+				// Transfer one vote to next available preference
+				let vote = numbered_votes.remove(&index).unwrap();
+				transfer_ballot(state, opts, elected_candidate, vote)?;
 				
-		// Transfer to next preference
+				index += skip_value;
+				if index >= total_ballots {
+					iteration += 1;
+					index = iteration + skip_value - 1;
+					
+					if iteration >= skip_value {
+						// We have run out of ballot papers
+						// Remaining ballot papers exhaust
+						
+						let surplus = &state.candidates[elected_candidate].votes - state.quota.as_ref().unwrap();
+						state.exhausted.transfer(&surplus);
+						state.candidates.get_mut(elected_candidate).unwrap().transfer(&-surplus);
+						break;
+					}
+				}
+			}
+		}
+	}
+	
+	return Ok(());
+}
+
+/// Transfer the given ballot paper to its next available preference, and check for candidates meeting the quota if --sample-per-ballot
+///
+/// Does nothing if --transferable-only and the ballot is nontransferable.
+fn transfer_ballot<'a, N: Number>(state: &mut CountState<'a, N>, opts: &STVOptions, elected_candidate: &Candidate, mut vote: Vote<'a, N>) -> Result<(), STVError> {
+	// Get next preference
 	let mut next_candidate = None;
 	for (i, preference) in vote.ballot.preferences.iter().enumerate().skip(vote.up_to_pref) {
 		let candidate = &state.election.candidates[*preference];
@@ -150,23 +338,6 @@ where
 		}
 	}
 	
-		index += skip_value;
-		if index >= total_ballots {
-			iteration += 1;
-			index = iteration + skip_value - 1;
-			
-			if iteration >= skip_value {
-				// We have run out of ballot papers
-				// Remaining ballot papers exhaust
-				
-				let surplus = &state.candidates[elected_candidate].votes - state.quota.as_ref().unwrap();
-				state.exhausted.transfer(&surplus);
-				state.candidates.get_mut(elected_candidate).unwrap().transfer(&-surplus);
-				break;
-			}
-		}
-	}
-	
 	return Ok(());
 }
 

src/stv/wasm.rs

@@ -229,6 +229,7 @@ impl STVOptions {
 		transferable_only: bool,
 		exclusion: &str,
 		meek_nz_exclusion: bool,
+		sample: &str,
 		sample_per_ballot: bool,
 		early_bulk_elect: bool,
 		bulk_exclude: bool,
@@ -256,6 +257,7 @@ impl STVOptions {
 			transferable_only,
 			exclusion.into(),
 			meek_nz_exclusion,
+			sample.into(),
 			sample_per_ballot,
 			early_bulk_elect,
 			bulk_exclude,

tests/cambridge.rs

@@ -29,6 +29,7 @@ fn cambridge_cc03_rational() {
 		.quota_criterion(stv::QuotaCriterion::GreaterOrEqual)
 		.surplus(stv::SurplusMethod::Cincinnati)
 		.transferable_only(true)
+		.sample(stv::SampleMethod::NthBallot)
 		.sample_per_ballot(true)
 		.early_bulk_elect(false)
 		.min_threshold("49".to_string())