From 07595d0ef9626e5f209a5679a1320a3d55caaf17 Mon Sep 17 00:00:00 2001
From: RunasSudo <runassudo@yingtongli.me>
Date: Tue, 17 Jan 2023 20:17:45 +1100
Subject: [PATCH] Use sql.js for processing on client side

---
 .gitignore      |  1 +
 export_db.sh    |  4 +++
 export_json.py  | 29 ---------------------
 html/index.html | 68 ++++++++++++++++++++++++++++---------------------
 import_pbs.py   | 53 +++++++++++++++++++++-----------------
 5 files changed, 73 insertions(+), 82 deletions(-)
 create mode 100755 export_db.sh
 delete mode 100644 export_json.py

diff --git a/.gitignore b/.gitignore
index 0aa3555..b6f034c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /data
 /database.db
+/html/database.db
 /html/pbs.json
diff --git a/export_db.sh b/export_db.sh
new file mode 100755
index 0000000..31972b7
--- /dev/null
+++ b/export_db.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+rm html/database.db
+sqlite3 database.db '.dump pbs_drug pbs_prescriber_type pbs_streamlined' | sqlite3 html/database.db
diff --git a/export_json.py b/export_json.py
deleted file mode 100644
index 66d5a6a..0000000
--- a/export_json.py
+++ /dev/null
@@ -1,29 +0,0 @@
-#   Copyright © 2023  Lee Yingtong Li (RunasSudo)
-#
-#   This program is free software: you can redistribute it and/or modify
-#   it under the terms of the GNU Affero General Public License as published by
-#   the Free Software Foundation, either version 3 of the License, or
-#   (at your option) any later version.
-#
-#   This program is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#   GNU Affero General Public License for more details.
-#
-#   You should have received a copy of the GNU Affero General Public License
-#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-import json
-import sqlite3
-
-con = sqlite3.connect('database.db')
-cur = con.cursor()
-
-results = []
-
-cur.execute('SELECT item_code, mp_pt, tpuu_or_mpp_pt, restriction_flag, mq, repeats, streamlined_authorities FROM pbs')
-for item_code, mp_pt, tpuu_or_mpp_pt, restriction_flag, mq, repeats, streamlined_authorities in cur.fetchall():
-	results.append(dict(item_code=item_code, mp_pt=mp_pt, tpuu_or_mpp_pt=tpuu_or_mpp_pt, restriction_flag=restriction_flag, mq=mq, repeats=repeats, streamlined_authorities=streamlined_authorities))
-
-with open('html/pbs.json', 'w') as f:
-	json.dump(results, f)
diff --git a/html/index.html b/html/index.html
index c3eb9e3..ef67377 100644
--- a/html/index.html
+++ b/html/index.html
@@ -57,51 +57,41 @@
 		<!--<script src="https://cdn.jsdelivr.net/npm/jquery@3.6.3/dist/jquery.min.js" integrity="sha256-pvPw+upLPUjgMXY0G+8O0xUf+/Im1MZjXxxgOcBQBXU=" crossorigin="anonymous"></script>-->
 		<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4" crossorigin="anonymous"></script>
 		<script src="autocomplete.js"></script>
+		<script src="https://cdn.jsdelivr.net/npm/sql.js@1.8.0/dist/sql-wasm.min.js"></script>
 		
 		<script>
-			var pbsData = null;
+			var db; // Keep in global namespace for debugging
 			
-			const xhr = new XMLHttpRequest();
-			xhr.addEventListener('load', function() {
-				pbsData = JSON.parse(xhr.responseText);
+			async function main() {
+				// Load SQLite database
+				const sqlPromise = initSqlJs({
+					locateFile: file => ('https://cdn.jsdelivr.net/npm/sql.js@1.8.0/dist/' + file)
+				});
+				const dataPromise = fetch('database.db').then(res => res.arrayBuffer());
+				const [SQL, buf] = await Promise.all([sqlPromise, dataPromise])
+				db = new SQL.Database(new Uint8Array(buf));
 				
 				// Initialise search bar
-				
-				const labels = [];
-				for (let row of pbsData) {
-					if (labels.indexOf(row['mp_pt']) < 0) {
-						labels.push(row['mp_pt']);
-					}
-				}
-				labels.sort();
-				
-				const data = [];
-				for (let label of labels) {
-					data.push({'label': label});
-				}
-				
+				const labels = execAsScalars(db.prepare('SELECT DISTINCT mp_pt FROM pbs_drug ORDER BY LOWER(mp_pt)'));
+				const data = labels.map(label => ({'label': label}));
 				const autocomplete = new Autocomplete(document.getElementById('search-input'), {
 					data: data,
 					maximumItems: 20,
 					threshold: 2,
 					onSelectItem: onClickSearchItem
 				});
-			});
-			xhr.open('GET', 'pbs.json');
-			xhr.send();
+			}
 			
 			function onClickSearchItem(item) {
 				// Find matching PBS items
-				const items = [];
-				for (let row of pbsData) {
-					if (row['mp_pt'] === item['label']) {
-						items.push(row);
-					}
-				}
+				const stmt = db.prepare('SELECT *, (SELECT COUNT(1) FROM pbs_streamlined WHERE pbs_drug.item_code = pbs_streamlined.item_code) AS streamlined_authorities FROM pbs_drug LEFT JOIN pbs_prescriber_type ON pbs_drug.item_code = pbs_prescriber_type.item_code WHERE LOWER(mp_pt) = ? AND prescriber_type = "M"');
+				stmt.bind([item.label.toLowerCase()]);
+				const items = execAsObjects(stmt);
+				
 				items.sort(comparePBSItems);
 				
 				// Update table
-				const tbody = document.querySelector(' search-results tbody');
+				const tbody = document.querySelector('#search-results tbody');
 				tbody.innerHTML = '';
 				for (let item of items) {
 					const tr = document.createElement('tr');
@@ -116,7 +106,7 @@
 						td = document.createElement('td'); td.innerHTML = '<a href="https://www.pbs.gov.au/medicine/item/' + item['item_code'] + '" target="_blank">Restricted</a>'; tr.appendChild(td);
 						tr.classList.add('table-warning');
 					} else if (item['restriction_flag'] === 'A') {
-						if (item['streamlined_authorities']) {
+						if (item['streamlined_authorities'] > 0) {
 							td = document.createElement('td'); td.innerHTML = '<a href="https://www.pbs.gov.au/medicine/item/' + item['item_code'] + '" target="_blank">Streamlined</a>'; tr.appendChild(td);
 							tr.classList.add('table-warning');
 						} else {
@@ -173,6 +163,26 @@
 				
 				return 0;
 			}
+			
+			function execAsScalars(stmt) {
+				let results = [];
+				while (stmt.step()) {
+					results.push(stmt.get()[0]);
+				}
+				stmt.free();
+				return results;
+			}
+			
+			function execAsObjects(stmt) {
+				let results = [];
+				while (stmt.step()) {
+					results.push(stmt.getAsObject());
+				}
+				stmt.free();
+				return results;
+			}
+			
+			main();
 		</script>
 	</body>
 </html>
diff --git a/import_pbs.py b/import_pbs.py
index 5b9f0e5..4a264a6 100644
--- a/import_pbs.py
+++ b/import_pbs.py
@@ -21,39 +21,44 @@ con = sqlite3.connect('database.db')
 cur = con.cursor()
 
 # Init schema
-cur.execute('DROP TABLE pbs')
-cur.execute('CREATE TABLE pbs (id INTEGER PRIMARY KEY AUTOINCREMENT, item_code CHARACTER(6), mp_pt TEXT, tpuu_or_mpp_pt TEXT, restriction_flag CHARACTER(1), mq INTEGER, repeats INTEGER, streamlined_authorities TEXT)')
+cur.execute('DROP TABLE IF EXISTS pbs_drug')
+cur.execute('CREATE TABLE pbs_drug (id INTEGER PRIMARY KEY AUTOINCREMENT, item_code CHARACTER(6), mp_pt TEXT, tpuu_or_mpp_pt TEXT, restriction_flag CHARACTER(1), mq INTEGER, repeats INTEGER)')
+
+cur.execute('DROP TABLE IF EXISTS pbs_prescriber_type')
+cur.execute('CREATE TABLE pbs_prescriber_type (id INTEGER PRIMARY KEY AUTOINCREMENT, item_code CHARACTER(6), prescriber_type CHARACTER(1))')
+
+cur.execute('DROP TABLE IF EXISTS pbs_streamlined')
+cur.execute('CREATE TABLE pbs_streamlined (id INTEGER PRIMARY KEY AUTOINCREMENT, item_code CHARACTER(6), treatment_of_code INTEGER)')
 
 # Read drug list, prescriber type
-with zipfile.ZipFile('2023-01-01-v3extracts.zip', 'r') as zipf:
+with zipfile.ZipFile('data/2023-01-01-v3extracts.zip', 'r') as zipf:
+	# drug_xxx.txt
+	
 	with zipf.open('drug_20230101.txt', 'r') as f:
 		df_drug = pd.read_csv(f, sep='!')
 	
+	for _, drug in df_drug.iterrows():
+		# Skip already added
+		cur.execute('SELECT COUNT(*) FROM pbs_drug WHERE item_code=?', (drug['item-code'],))
+		if cur.fetchone()[0] > 0:
+			continue
+		
+		cur.execute('INSERT INTO pbs_drug (item_code, mp_pt, tpuu_or_mpp_pt, restriction_flag, mq, repeats) VALUES (?, ?, ?, ?, ?, ?)', (drug['item-code'], drug['mp-pt'], drug['tpuu-or-mpp-pt'], drug['restriction-flag'], drug['mq'], drug['repeats']))
+	
+	# Prescriber_type_xxx.txt
+	
 	with zipf.open('Prescriber_type_20230101.txt', 'r') as f:
 		df_prescriber_type = pd.read_csv(f, sep='\t', header=0, names=['mp-pt', 'item-code', 'prescriber-type'])
-
-df_drug = df_drug.merge(df_prescriber_type[['item-code', 'prescriber-type']], how='left', on='item-code')
-
-# Filter only drugs able to be prescribed by medical practitioners
-df_drug = df_drug[df_drug['prescriber-type'] == 'M']
-
-for _, drug in df_drug[['item-code', 'mp-pt', 'tpuu-or-mpp-pt', 'restriction-flag', 'mq', 'repeats']].iterrows():
-	# Skip already added
-	cur.execute('SELECT COUNT(*) FROM pbs WHERE item_code=?', (drug['item-code'],))
-	if cur.fetchone()[0] > 0:
-		continue
 	
-	# Add to SQL
-	cur.execute('INSERT INTO pbs (item_code, mp_pt, tpuu_or_mpp_pt, restriction_flag, mq, repeats) VALUES (?, ?, ?, ?, ?, ?)', (drug['item-code'], drug['mp-pt'], drug['tpuu-or-mpp-pt'], drug['restriction-flag'], drug['mq'], drug['repeats']))
-
-# Read streamlined authorities
-with zipfile.ZipFile('2023-01-01-v3extracts.zip', 'r') as zipf:
+	for _, prescriber_type in df_prescriber_type.iterrows():
+		cur.execute('INSERT INTO pbs_prescriber_type (item_code, prescriber_type) VALUES (?, ?)', (prescriber_type['item-code'], prescriber_type['prescriber-type']))
+	
+	# streamlined_xxx.txt (streamlined authorities)
+	
 	with zipf.open('streamlined_20230101.txt', 'r') as f:
 		df_streamlined = pd.read_csv(f, sep='\t')
-
-df_streamlined = df_drug.merge(df_streamlined[['item-code', 'treatment-of-code']], how='inner', on='item-code')
-
-for k, v in df_streamlined.groupby('item-code'):
-	cur.execute('UPDATE pbs SET streamlined_authorities=? WHERE item_code=?', (','.join(v['treatment-of-code'].astype(str)), k))
+	
+	for _, streamlined in df_streamlined.iterrows():
+		cur.execute('INSERT INTO pbs_streamlined (item_code, treatment_of_code) VALUES (?, ?)', (streamlined['item-code'], streamlined['treatment-of-code']))
 
 con.commit()