Implement serialised binary format

Cargo.lock

@@ -1,11 +1,24 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "adler"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "ahash"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98"
+dependencies = [
+ "getrandom",
+ "once_cell",
+ "version_check",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.18"
@@ -80,6 +93,27 @@ version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
 
+[[package]]
+name = "bytecheck"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb738a1e65989ecdcd5bba16079641bd7209688fa546e1064832fd6e012fd32a"
+dependencies = [
+ "bytecheck_derive",
+ "ptr_meta",
+]
+
+[[package]]
+name = "bytecheck_derive"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3b4dff26fdc9f847dab475c9fec16f2cba82d5aa1f09981b87c44520721e10a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "byteorder"
 version = "1.4.3"
@@ -329,6 +363,17 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getrandom"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
+dependencies = [
+ "cfg-if 1.0.0",
+ "libc",
+ "wasi",
+]
+
 [[package]]
 name = "git-version"
 version = "0.3.4"
@@ -367,6 +412,15 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+dependencies = [
+ "ahash",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.2"
@@ -402,7 +456,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
 dependencies = [
  "autocfg",
- "hashbrown",
+ "hashbrown 0.9.1",
 ]
 
 [[package]]
@@ -545,6 +599,12 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "once_cell"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
+
 [[package]]
 name = "opaque-debug"
 version = "0.3.0"
@@ -572,6 +632,7 @@ dependencies = [
  "num-traits",
  "paste",
  "predicates",
+ "rkyv",
  "rug",
  "sha2",
  "utf8-chars",
@@ -668,6 +729,26 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "ptr_meta"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"
+dependencies = [
+ "ptr_meta_derive",
+]
+
+[[package]]
+name = "ptr_meta_derive"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.9"
@@ -724,6 +805,40 @@ version = "0.6.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
 
+[[package]]
+name = "rend"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d0351a2e529ee30d571ef31faa5a4e0b9addaad087697b77efb20d2809e41c7"
+dependencies = [
+ "bytecheck",
+]
+
+[[package]]
+name = "rkyv"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e804c561b577f5836dc8a1962b7f7a03eae36f716dcd5f779c5d52a0e9c09a7"
+dependencies = [
+ "bytecheck",
+ "hashbrown 0.11.2",
+ "ptr_meta",
+ "rend",
+ "rkyv_derive",
+ "seahash",
+]
+
+[[package]]
+name = "rkyv_derive"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0afbc272334d4a4896e382508531f941a7d9505057d7424bcbed653682ce661e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "rug"
 version = "1.12.0"
@@ -750,6 +865,12 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
+[[package]]
+name = "seahash"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
+
 [[package]]
 name = "semver"
 version = "0.11.0"
@@ -876,6 +997,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "wasi"
+version = "0.10.2+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.74"

Cargo.toml

@@ -32,20 +32,21 @@ paste = "1.0.5"
 assert_cmd = "1.0.5"
 csv = "1.1.6"
 flate2 = "1.0"
+rkyv = "0.7.15"
 utf8-chars = "1.0.2"
 xmltree = "0.10.3"
 
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies.clap]
+#version = "3.0.0-beta.4" # proc-macro2 version conflict with rkyv
+git = "https://github.com/clap-rs/clap"
+branch = "master"
+default-features = false
+features = ["std", "derive"]
+
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies.rug]
 version = "1.12"
 default-features = false
 features = ["integer", "rational", "float"]
 
-[target.'cfg(not(target_arch = "wasm32"))'.dependencies.clap]
-#version = "3.0.0-beta.2" # Bug 2279
-git = "https://github.com/clap-rs/clap"
-branch = "master"
-default-features = false
-features = ["std", "derive"]
-
 [profile.test]
 opt-level = 3

src/cli/convert.rs

@@ -37,11 +37,11 @@ pub struct SubcmdOptions {
 	outfile: String,
 	
 	/// Format of input file
-	#[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["blt", "csp"], value_name="format")]
+	#[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["bin", "blt", "csp"], value_name="format")]
 	r#in: Option<String>,
 	
 	/// Format of output file
-	#[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["blt", "csp"], value_name="format")]
+	#[clap(help_heading=Some("INPUT/OUTPUT"), short, long, possible_values=&["bin", "blt", "csp"], value_name="format")]
 	out: Option<String>,
 	
 	/// Number of seats
@@ -53,7 +53,9 @@ pub struct SubcmdOptions {
 pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> {
 	// Auto-detect input/output formats
 	if cmd_opts.r#in == None {
-		if cmd_opts.infile.ends_with(".blt") {
+		if cmd_opts.infile.ends_with(".bin") {
+			cmd_opts.r#in = Some("bin".to_string());
+		} else if cmd_opts.infile.ends_with(".blt") {
 			cmd_opts.r#in = Some("blt".to_string());
 		} else if cmd_opts.infile.ends_with(".csp") {
 			cmd_opts.r#in = Some("csp".to_string());
@@ -63,7 +65,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> {
 		}
 	}
 	if cmd_opts.out == None {
-		if cmd_opts.outfile.ends_with(".blt") {
+		if cmd_opts.outfile.ends_with(".bin") {
+			cmd_opts.out = Some("bin".to_string());
+		} else if cmd_opts.outfile.ends_with(".blt") {
 			cmd_opts.out = Some("blt".to_string());
 		} else if cmd_opts.outfile.ends_with(".csp") {
 			cmd_opts.out = Some("csp".to_string());
@@ -77,6 +81,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> {
 	let mut election: Election<Rational>;
 	
 	match cmd_opts.r#in.as_deref().unwrap() {
+		"bin" => {
+			election = parser::bin::parse_path(cmd_opts.infile);
+		}
 		"blt" => {
 			match parser::blt::parse_path(cmd_opts.infile) {
 				Ok(e) => {
@@ -111,6 +118,9 @@ pub fn main(mut cmd_opts: SubcmdOptions) -> Result<(), i32> {
 	let output = File::create(cmd_opts.outfile).expect("IO Error");
 	
 	match cmd_opts.out.as_deref().unwrap() {
+		"bin" => {
+			writer::bin::write(election, output);
+		}
 		"blt" => {
 			writer::blt::write(election, output);
 		}

src/cli/stv.rs

@@ -18,7 +18,7 @@
 use crate::constraints::Constraints;
 use crate::election::{CandidateState, CountState, Election};
 use crate::numbers::{Fixed, GuardedFixed, NativeFloat64, Number, Rational};
-use crate::parser::blt;
+use crate::parser::{bin, blt};
 use crate::stv::{self, STVOptions};
 use crate::ties;
 
@@ -37,8 +37,13 @@ pub struct SubcmdOptions {
 	// -- File input --
 	
 	/// Path to the BLT file to be counted
+	#[clap(help_heading=Some("INPUT"))]
 	filename: String,
 	
+	/// Input is in serialised binary format from "opentally convert"
+	#[clap(help_heading=Some("INPUT"), long)]
+	bin: bool,
+	
 	// ----------------------
 	// -- Numbers settings --
 	
@@ -189,25 +194,25 @@ pub struct SubcmdOptions {
 pub fn main(cmd_opts: SubcmdOptions) -> Result<(), i32> {
 	// Read and count election according to --numbers
 	if cmd_opts.numbers == "rational" {
-		let mut election = election_from_file(&cmd_opts.filename)?;
+		let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?;
 		maybe_load_constraints(&mut election, &cmd_opts.constraints);
 		
 		// Must specify ::<N> here and in a few other places because ndarray causes E0275 otherwise
 		count_election::<Rational>(election, cmd_opts)?;
 	} else if cmd_opts.numbers == "float64" {
-		let mut election = election_from_file(&cmd_opts.filename)?;
+		let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?;
 		maybe_load_constraints(&mut election, &cmd_opts.constraints);
 		count_election::<NativeFloat64>(election, cmd_opts)?;
 	} else if cmd_opts.numbers == "fixed" {
 		Fixed::set_dps(cmd_opts.decimals);
 		
-		let mut election = election_from_file(&cmd_opts.filename)?;
+		let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?;
 		maybe_load_constraints(&mut election, &cmd_opts.constraints);
 		count_election::<Fixed>(election, cmd_opts)?;
 	} else if cmd_opts.numbers == "gfixed" {
 		GuardedFixed::set_dps(cmd_opts.decimals);
 		
-		let mut election = election_from_file(&cmd_opts.filename)?;
+		let mut election = election_from_file(&cmd_opts.filename, cmd_opts.bin)?;
 		maybe_load_constraints(&mut election, &cmd_opts.constraints);
 		count_election::<GuardedFixed>(election, cmd_opts)?;
 	}
@@ -215,7 +220,12 @@ pub fn main(cmd_opts: SubcmdOptions) -> Result<(), i32> {
 	return Ok(());
 }
 
-fn election_from_file<N: Number>(path: &str) -> Result<Election<N>, i32> {
+fn election_from_file<N: Number>(path: &str, bin: bool) -> Result<Election<N>, i32> {
+	if bin {
+		// BIN format
+		return Ok(bin::parse_path(path));
+	} else {
+		// BLT format
 		match blt::parse_path(path) {
 			Ok(e) => return Ok(e),
 			Err(err) => {
@@ -224,6 +234,7 @@ fn election_from_file<N: Number>(path: &str) -> Result<Election<N>, i32> {
 			}
 		}
 	}
+}
 
 fn maybe_load_constraints<N: Number>(election: &mut Election<N>, constraints: &Option<String>) {
 	if let Some(c) = constraints {

src/constraints.rs

@@ -22,12 +22,16 @@ use crate::stv::{ConstraintMode, STVOptions};
 use itertools::Itertools;
 use ndarray::{Array, Dimension, IxDyn};
 
+#[cfg(not(target_arch = "wasm32"))]
+use rkyv::{Archive, Deserialize, Serialize};
+
 use std::collections::HashMap;
 use std::fmt;
 use std::ops;
 
 /// Constraints for an [crate::election::Election]
 #[derive(Debug)]
+#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
 pub struct Constraints(pub Vec<Constraint>);
 
 impl Constraints {
@@ -121,6 +125,7 @@ impl Constraints {
 
 /// A single dimension of constraint
 #[derive(Debug)]
+#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
 pub struct Constraint {
 	/// Name of this constraint
 	pub name: String,
@@ -130,6 +135,7 @@ pub struct Constraint {
 
 /// A group of candidates, of which a certain minimum and maximum must be elected
 #[derive(Debug)]
+#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
 pub struct ConstrainedGroup {
 	/// Name of this group
 	pub name: String,

src/election.rs

@@ -21,10 +21,14 @@ use crate::numbers::Number;
 use crate::sharandom::SHARandom;
 use crate::stv::{QuotaMode, STVOptions};
 
+#[cfg(not(target_arch = "wasm32"))]
+use rkyv::{Archive, Archived, Deserialize, Fallible, Resolver, Serialize, with::{ArchiveWith, DeserializeWith, SerializeWith}};
+
 use std::cmp::max;
 use std::collections::HashMap;
 
 /// An election to be counted
+#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
 pub struct Election<N> {
 	/// Name of the election
 	pub name: String,
@@ -62,6 +66,7 @@ impl<N: Number> Election<N> {
 
 /// A candidate in an [Election]
 #[derive(PartialEq, Eq, Hash)]
+#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
 pub struct Candidate {
 	/// Name of the candidate
 	pub name: String,
@@ -363,13 +368,43 @@ pub struct Vote<'a, N> {
 }
 
 /// A record of a voter's preferences
+#[cfg_attr(not(target_arch = "wasm32"), derive(Archive, Deserialize, Serialize))]
 pub struct Ballot<N> {
 	/// Original value/weight of the ballot
+	#[cfg_attr(not(target_arch = "wasm32"), with(SerializedNum))]
 	pub orig_value: N,
 	/// Indexes of candidates preferenced on the ballot
 	pub preferences: Vec<usize>,
 }
 
+/// rkyv-serialized representation of [Number]
+#[cfg(not(target_arch = "wasm32"))]
+pub struct SerializedNum;
+
+#[cfg(not(target_arch = "wasm32"))]
+impl<N: Number> ArchiveWith<N> for SerializedNum {
+	type Archived = Archived<String>;
+	type Resolver = Resolver<String>;
+	
+	unsafe fn resolve_with(field: &N, pos: usize, resolver: Self::Resolver, out: *mut Self::Archived) {
+		field.to_string().resolve(pos, resolver, out);
+	}
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl<N: Number, S: Fallible + ?Sized> SerializeWith<N, S> for SerializedNum where String: Serialize<S> {
+	fn serialize_with(field: &N, serializer: &mut S) -> Result<Self::Resolver, S::Error> {
+		return field.to_string().serialize(serializer);
+	}
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl<N: Number, D: Fallible + ?Sized> DeserializeWith<Archived<String>, N, D> for SerializedNum where Archived<String>: Deserialize<String, D> {
+	fn deserialize_with(field: &Archived<String>, deserializer: &mut D) -> Result<N, D::Error> {
+		return Ok(N::parse(&field.deserialize(deserializer)?));
+	}
+}
+
 /// State of a [Candidate] during a count
 #[allow(dead_code)]
 #[derive(PartialEq)]

src/parser/bin.rs

@@ -0,0 +1,33 @@
+/*  OpenTally: Open-source election vote counting
+ *  Copyright © 2021 Lee Yingtong Li (RunasSudo)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use crate::election::Election;
+use crate::numbers::Number;
+
+use rkyv::{Deserialize, Infallible, archived_root};
+
+use std::fs;
+use std::path::Path;
+
+/// Parse the given BIN file
+pub fn parse_path<P: AsRef<Path>, N: Number>(path: P) -> Election<N> {
+	let content = fs::read(path).expect("IO Error");
+	let archived = unsafe {
+		archived_root::<Election<N>>(&content)
+	};
+	return archived.deserialize(&mut Infallible).unwrap();
+}

src/parser/mod.rs

@@ -15,6 +15,10 @@
  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+/// BIN file parser
+#[cfg(not(target_arch = "wasm32"))]
+pub mod bin;
+
 /// BLT file parser
 pub mod blt;
 /// CSP file parser

src/writer/bin.rs

@@ -0,0 +1,35 @@
+/*  OpenTally: Open-source election vote counting
+ *  Copyright © 2021 Lee Yingtong Li (RunasSudo)
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+use crate::election::Election;
+use crate::numbers::Number;
+
+use rkyv::ser::{Serializer, serializers::AllocSerializer};
+
+use std::io::{BufWriter, Write};
+
+/// Write the [Election] into BIN format
+pub fn write<W: Write, N: Number>(election: Election<N>, output: W) {
+	// Serialize data using rkyv
+	let mut serializer = AllocSerializer::<256>::default();
+	serializer.serialize_value(&election).unwrap();
+	let buffer = serializer.into_serializer().into_inner();
+	
+	// Write output
+	let mut output = BufWriter::new(output);
+	output.write(&buffer).expect("IO Error");
+}

src/writer/mod.rs

@@ -15,6 +15,10 @@
  *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+/// BIN file writer
+#[cfg(not(target_arch = "wasm32"))]
+pub mod bin;
+
 /// BLT file writer
 pub mod blt;
 /// CSP file writer