Optimise next_preferences algorithm to avoid re-examining earlier preferences

Senate count from 9.02 to 8.22s

pyRCV2/method/base_stv.py

@@ -16,7 +16,7 @@
 
 __pragma__ = lambda x: None
 
-from pyRCV2.model import CandidateState, CountCard, CountCompleted, CountStepResult
+from pyRCV2.model import BallotInCount, CandidateState, CountCard, CountCompleted, CountStepResult
 import pyRCV2.numbers
 from pyRCV2.numbers import Num
 from pyRCV2.safedict import SafeDict
@@ -101,15 +101,18 @@ class BaseSTVCounter:
 		
 		for ballot in self.election.ballots:
 			__pragma__('opov')
-			candidate = next((c for c in ballot.preferences if self.candidates[c].state == CandidateState.HOPEFUL), None)
+			for i, candidate in enumerate(ballot.preferences):
+				count_card = self.candidates[candidate]
 				
-			if candidate is not None:
+				if count_card.state == CandidateState.HOPEFUL:
-				self.candidates[candidate].transfers += ballot.value
+					count_card.transfers += ballot.value
-				if len(self.candidates[candidate].parcels) == 0:
+					if len(count_card.parcels) == 0:
-					self.candidates[candidate].parcels.append([(ballot, Num(ballot.value))])
+						count_card.parcels.append([BallotInCount(ballot, Num(ballot.value), i)])
 					else:
-					self.candidates[candidate].parcels[0].append((ballot, Num(ballot.value)))
+						count_card.parcels[0].append(BallotInCount(ballot, Num(ballot.value), i))
+					break
 			else:
+				# No available preference
 				self.exhausted.transfers += ballot.value
 				#self.exhausted.parcels[0].append((ballot, Num(ballot.value)))
 			__pragma__('noopov')
@@ -601,7 +604,7 @@ class BaseSTVCounter:
 		"""
 		Examine the specified ballots and group ballot papers by next available preference
 		"""
-		# SafeDict: Candidate -> [List[Ballot], ballots, votes]
+		# SafeDict: Candidate -> [List[BallotInCount], ballots, votes]
 		next_preferences = SafeDict([(c, [[], Num('0'), Num('0')]) for c, cc in self.candidates.items()])
 		total_ballots = Num('0')
 		total_votes = Num('0')
@@ -611,21 +614,25 @@ class BaseSTVCounter:
 		exhausted_votes = Num('0')
 		
 		for parcel in parcels:
-			for ballot, ballot_value in parcel:
+			for bc in parcel:
 				__pragma__('opov')
-				total_ballots += ballot.value
+				total_ballots += bc.ballot.value
-				total_votes += ballot_value
+				total_votes += bc.ballot_value
-				candidate = next((c for c in ballot.preferences if self.candidates[c].state == CandidateState.HOPEFUL), None)
 				
-				if candidate is not None:
+				for i in range(bc.last_preference + 1, len(bc.ballot.preferences)):
-					next_preferences[candidate][0].append((ballot, ballot_value))
+					candidate = bc.ballot.preferences[i]
-					next_preferences[candidate][1] += ballot.value
+					count_card = self.candidates[candidate]
-					next_preferences[candidate][2] += ballot_value
+					
+					if count_card.state == CandidateState.HOPEFUL:
+						next_preferences[candidate][0].append(BallotInCount(bc.ballot, bc.ballot_value, i))
+						next_preferences[candidate][1] += bc.ballot.value
+						next_preferences[candidate][2] += bc.ballot_value
+						break
 				else:
-					next_exhausted.append((ballot, ballot_value))
+					# No next available preference
-					exhausted_ballots += ballot.value
+					next_exhausted.append(bc)
-					exhausted_votes += ballot_value
+					exhausted_ballots += bc.ballot.value
-				
+					exhausted_votes += bc.ballot_value
 				__pragma__('noopov')
 		
 		return next_preferences, total_ballots, total_votes, next_exhausted, exhausted_ballots, exhausted_votes

pyRCV2/method/gregory.py

@@ -119,37 +119,38 @@ class BaseGregorySTVCounter(BaseSTVCounter):
 					self.candidates[candidate].transfers += self.round_votes(num_units * tv)
 			__pragma__('noopov')
 			
-			for ballot, ballot_value in cand_ballots:
+			for bc in cand_ballots:
 				__pragma__('opov')
 				if self.options['papers'] == 'transferable':
 					if transferable_votes > surplus:
 						if self.options['round_tvs'] is None:
 							if is_weighted:
-								new_value = (ballot_value * surplus) / transferable_units
+								new_value = (bc.ballot_value * surplus) / transferable_units
 							else:
-								new_value = (ballot.value * surplus) / transferable_units
+								new_value = (bc.ballot.value * surplus) / transferable_units
 						else:
 							tv = self.round_tv(surplus / transferable_units)
 							if is_weighted:
-								new_value = ballot_value * tv
+								new_value = bc.ballot_value * tv
 							else:
-								new_value = ballot.value * tv
+								new_value = bc.ballot.value * tv
 					else:
-						new_value = ballot_value
+						new_value = bc.ballot_value
 				else:
 					if self.options['round_tvs'] is None:
 						if is_weighted:
-							new_value = (ballot_value * surplus) / total_units
+							new_value = (bc.ballot_value * surplus) / total_units
 						else:
-							new_value = (ballot.value * surplus) / total_units
+							new_value = (bc.ballot.value * surplus) / total_units
 					else:
 						tv = self.round_tv(surplus / total_units)
 						if is_weighted:
-							new_value = ballot_value * tv
+							new_value = bc.ballot_value * tv
 						else:
-							new_value = ballot.value * tv
+							new_value = bc.ballot.value * tv
 				
-				new_parcel.append((ballot, self.round_weight(new_value)))
+				bc.ballot_value = self.round_weight(new_value)
+				new_parcel.append(bc)
 				__pragma__('noopov')
 		
 		__pragma__('opov')
@@ -174,29 +175,29 @@ class BaseGregorySTVCounter(BaseSTVCounter):
 	def do_exclusion(self, candidates_excluded):
 		"""Implements BaseSTVCounter.do_exclusion"""
 		# Optimisation: Pre-sort exclusion ballots if applicable
-		# self._exclusion[1] -> list of ballots-per-stage, ballots-per-stage = List[Tuple[Candidate,List[Ballot+Value]]]
+		# self._exclusion[1] -> list of ballots-per-stage ; ballots-per-stage = List[Tuple[Candidate,List[BallotInCount]]]
 		if self._exclusion is None:
 			if self.options['exclusion'] == 'one_round':
-				self._exclusion = (candidates_excluded, [[(c, [b for p in cc.parcels for b in p]) for c, cc in candidates_excluded]])
+				self._exclusion = (candidates_excluded, [[(c, [bc for p in cc.parcels for bc in p]) for c, cc in candidates_excluded]])
 			elif self.options['exclusion'] == 'parcels_by_order':
 				c, cc = candidates_excluded[0]
 				self._exclusion = (candidates_excluded, [[(c, p)] for p in cc.parcels])
 			elif self.options['exclusion'] == 'by_value':
-				ballots = [(c, b, bv) for c, cc in candidates_excluded for p in cc.parcels for b, bv in p]
+				ballots = [(c, bc) for c, cc in candidates_excluded for p in cc.parcels for bc in p]
 				
 				# Sort ballots by value
 				__pragma__('opov')
-				ballots.sort(key=lambda x: x[2] / x[1].value, reverse=True)
+				ballots.sort(key=lambda x: x[1].ballot_value / x[1].ballot.value, reverse=True)
+				if self.options['round_tvs']:
+					ballots_by_value = groupby(ballots, lambda x: self.round_tv(x[1].ballot_value / x[1].ballot.value))
+				else:
 					# Round to 8 decimal places to consider equality
 					# FIXME: Work out a better way of doing this
-				if self.options['round_tvs']:
+					ballots_by_value = groupby(ballots, lambda x: (x[1].ballot_value / x[1].ballot.value).round(8, x[1].ballot_value.ROUND_DOWN))
-					ballots_by_value = groupby(ballots, lambda x: self.round_tv(x[2] / x[1].value))
-				else:
-					ballots_by_value = groupby(ballots, lambda x: (x[2] / x[1].value).round(8, x[2].ROUND_DOWN))
 				__pragma__('noopov')
 				
 				# TODO: Can we combine ballots for each candidate within each stage?
-				self._exclusion = (candidates_excluded, [[(c, [(b, bv)]) for c, b, bv in x] for x in ballots_by_value])
+				self._exclusion = (candidates_excluded, [[(c, [bc]) for c, bc in x] for x in ballots_by_value])
 			else:
 				raise STVException('Invalid exclusion mode') # pragma: no cover
 		
@@ -205,11 +206,11 @@ class BaseGregorySTVCounter(BaseSTVCounter):
 		
 		# Transfer votes
 		
-		next_preferences, total_ballots, total_votes, next_exhausted, exhausted_ballots, exhausted_votes = self.next_preferences([bb for c, bb in this_exclusion])
+		next_preferences, total_ballots, total_votes, next_exhausted, exhausted_ballots, exhausted_votes = self.next_preferences([bc for c, bc in this_exclusion])
 		
 		if self.options['exclusion'] != 'one_round':
 			__pragma__('opov')
-			self.logs.append('Transferring ' + total_ballots.pp(0) + ' ballot papers, totalling ' + total_votes.pp(2) + ' votes, received at value ' + (this_exclusion[0][1][0][1] / this_exclusion[0][1][0][0].value).pp(2) + '.')
+			self.logs.append('Transferring ' + total_ballots.pp(0) + ' ballot papers, totalling ' + total_votes.pp(2) + ' votes, received at value ' + (this_exclusion[0][1][0].ballot_value / this_exclusion[0][1][0].ballot.value).pp(2) + '.')
 			__pragma__('noopov')
 		
 		for candidate, x in next_preferences.items():
@@ -225,9 +226,9 @@ class BaseGregorySTVCounter(BaseSTVCounter):
 			self.candidates[candidate].transfers += self.round_votes(num_votes)
 			__pragma__('noopov')
 			
-			for ballot, ballot_value in cand_ballots:
+			for bc in cand_ballots:
 				__pragma__('opov')
-				new_parcel.append((ballot, ballot_value))
+				new_parcel.append(bc)
 				__pragma__('noopov')
 		
 		# Subtract votes
@@ -238,9 +239,9 @@ class BaseGregorySTVCounter(BaseSTVCounter):
 		
 		for candidate, ballots in this_exclusion:
 			total_votes = Num(0)
-			for ballot, ballot_value in ballots:
+			for bc in ballots:
 				__pragma__('opov')
-				total_votes += ballot_value
+				total_votes += bc.ballot_value
 				__pragma__('noopov')
 			
 			__pragma__('opov')

pyRCV2/method/meek.py

@@ -33,7 +33,7 @@ class MeekCountCard(CountCard):
 		result = MeekCountCard()
 		result.orig_votes = self.orig_votes
 		result.transfers = self.transfers
-		result.parcels = [[(b[0].clone(), b[1]) for b in p] for p in self.parcels]
+		#result.parcels = [[(b[0].clone(), b[1]) for b in p] for p in self.parcels]
 		result.state = self.state
 		result.order_elected = self.order_elected
 		result.keep_value = keep_value

pyRCV2/model.py

@@ -38,6 +38,10 @@ class CandidateState:
 	WITHDRAWN = 60
 
 class Ballot:
+	"""
+	Represents a voter's (or if weighted, multiple voters') preferences
+	"""
+	
 	def __init__(self, value, preferences):
 		self.value = value
 		self.preferences = preferences
@@ -45,6 +49,20 @@ class Ballot:
 	def clone(self):
 		return Ballot(self.value, self.preferences)
 
+class BallotInCount:
+	"""
+	Represents a Ballot held by a candidate at a particular value during the count
+	"""
+	
+	__slots__ = ['ballot', 'ballot_value', 'last_preference']
+	
+	def __init__(self, ballot, ballot_value, last_preference):
+		self.ballot = ballot
+		self.ballot_value = ballot_value
+		
+		# Optimisation: Record the most-recently used preference so earlier preferences do not need to be later examined
+		self.last_preference = last_preference
+
 class Election:
 	"""
 	Represents a BLT election