RunasSudo/MedicineSearch
1
Fork 0
Code Issues 2 Pull Requests Releases Activity

Use sql.js for processing on client side

Browse Source
This commit is contained in:
RunasSudo 2023-01-17 20:17:45 +11:00
parent dd2665bbe9
commit 07595d0ef9
Signed by: RunasSudo
GPG Key ID: 7234E476BF21C61A
5 changed files with 73 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
/data
/database.db
/html/database.db
/html/pbs.json

4
export_db.sh Executable file
View File

@ -0,0 +1,4 @@
#!/bin/bash
rm html/database.db
sqlite3 database.db '.dump pbs_drug pbs_prescriber_type pbs_streamlined' | sqlite3 html/database.db

29
export_json.py
View File

@ -1,29 +0,0 @@
# Copyright © 2023 Lee Yingtong Li (RunasSudo)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import json
import sqlite3
con = sqlite3.connect('database.db')
cur = con.cursor()
results = []
cur.execute('SELECT item_code, mp_pt, tpuu_or_mpp_pt, restriction_flag, mq, repeats, streamlined_authorities FROM pbs')
for item_code, mp_pt, tpuu_or_mpp_pt, restriction_flag, mq, repeats, streamlined_authorities in cur.fetchall():
results.append(dict(item_code=item_code, mp_pt=mp_pt, tpuu_or_mpp_pt=tpuu_or_mpp_pt, restriction_flag=restriction_flag, mq=mq, repeats=repeats, streamlined_authorities=streamlined_authorities))
with open('html/pbs.json', 'w') as f:
json.dump(results, f)

68
html/index.html
View File

@ -57,51 +57,41 @@
<!--<script src="https://cdn.jsdelivr.net/npm/jquery@3.6.3/dist/jquery.min.js" integrity="sha256-pvPw+upLPUjgMXY0G+8O0xUf+/Im1MZjXxxgOcBQBXU=" crossorigin="anonymous"></script>-->
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-kenU1KFdBIe4zVF0s0G1M5b4hcpxyD9F7jL+jjXkk+Q2h455rYXK/7HAuoJl+0I4" crossorigin="anonymous"></script>
<script src="autocomplete.js"></script>
<script src="https://cdn.jsdelivr.net/npm/sql.js@1.8.0/dist/sql-wasm.min.js"></script>
<script>
var pbsData = null;
var db; // Keep in global namespace for debugging
const xhr = new XMLHttpRequest();
xhr.addEventListener('load', function() {
pbsData = JSON.parse(xhr.responseText);
async function main() {
// Load SQLite database
const sqlPromise = initSqlJs({
locateFile: file => ('https://cdn.jsdelivr.net/npm/sql.js@1.8.0/dist/' + file)
});
const dataPromise = fetch('database.db').then(res => res.arrayBuffer());
const [SQL, buf] = await Promise.all([sqlPromise, dataPromise])
db = new SQL.Database(new Uint8Array(buf));
// Initialise search bar
const labels = [];
for (let row of pbsData) {
if (labels.indexOf(row['mp_pt']) < 0) {
labels.push(row['mp_pt']);
}
}
labels.sort();
const data = [];
for (let label of labels) {
data.push({'label': label});
}
const labels = execAsScalars(db.prepare('SELECT DISTINCT mp_pt FROM pbs_drug ORDER BY LOWER(mp_pt)'));
const data = labels.map(label => ({'label': label}));
const autocomplete = new Autocomplete(document.getElementById('search-input'), {
data: data,
maximumItems: 20,
threshold: 2,
onSelectItem: onClickSearchItem
});
});
xhr.open('GET', 'pbs.json');
xhr.send();
}
function onClickSearchItem(item) {
// Find matching PBS items
const items = [];
for (let row of pbsData) {
if (row['mp_pt'] === item['label']) {
items.push(row);
}
}
const stmt = db.prepare('SELECT *, (SELECT COUNT(1) FROM pbs_streamlined WHERE pbs_drug.item_code = pbs_streamlined.item_code) AS streamlined_authorities FROM pbs_drug LEFT JOIN pbs_prescriber_type ON pbs_drug.item_code = pbs_prescriber_type.item_code WHERE LOWER(mp_pt) = ? AND prescriber_type = "M"');
stmt.bind([item.label.toLowerCase()]);
const items = execAsObjects(stmt);
items.sort(comparePBSItems);
// Update table
const tbody = document.querySelector(' search-results tbody');
const tbody = document.querySelector('#search-results tbody');
tbody.innerHTML = '';
for (let item of items) {
const tr = document.createElement('tr');
@ -116,7 +106,7 @@
td = document.createElement('td'); td.innerHTML = '<a href="https://www.pbs.gov.au/medicine/item/' + item['item_code'] + '" target="_blank">Restricted</a>'; tr.appendChild(td);
tr.classList.add('table-warning');
} else if (item['restriction_flag'] === 'A') {
if (item['streamlined_authorities']) {
if (item['streamlined_authorities'] > 0) {
td = document.createElement('td'); td.innerHTML = '<a href="https://www.pbs.gov.au/medicine/item/' + item['item_code'] + '" target="_blank">Streamlined</a>'; tr.appendChild(td);
tr.classList.add('table-warning');
} else {
@ -173,6 +163,26 @@
return 0;
}
function execAsScalars(stmt) {
let results = [];
while (stmt.step()) {
results.push(stmt.get()[0]);
}
stmt.free();
return results;
}
function execAsObjects(stmt) {
let results = [];
while (stmt.step()) {
results.push(stmt.getAsObject());
}
stmt.free();
return results;
}
main();
</script>
</body>
</html>

47
import_pbs.py
View File

@ -21,39 +21,44 @@ con = sqlite3.connect('database.db')
cur = con.cursor()
# Init schema
cur.execute('DROP TABLE pbs')
cur.execute('CREATE TABLE pbs (id INTEGER PRIMARY KEY AUTOINCREMENT, item_code CHARACTER(6), mp_pt TEXT, tpuu_or_mpp_pt TEXT, restriction_flag CHARACTER(1), mq INTEGER, repeats INTEGER, streamlined_authorities TEXT)')
cur.execute('DROP TABLE IF EXISTS pbs_drug')
cur.execute('CREATE TABLE pbs_drug (id INTEGER PRIMARY KEY AUTOINCREMENT, item_code CHARACTER(6), mp_pt TEXT, tpuu_or_mpp_pt TEXT, restriction_flag CHARACTER(1), mq INTEGER, repeats INTEGER)')
cur.execute('DROP TABLE IF EXISTS pbs_prescriber_type')
cur.execute('CREATE TABLE pbs_prescriber_type (id INTEGER PRIMARY KEY AUTOINCREMENT, item_code CHARACTER(6), prescriber_type CHARACTER(1))')
cur.execute('DROP TABLE IF EXISTS pbs_streamlined')
cur.execute('CREATE TABLE pbs_streamlined (id INTEGER PRIMARY KEY AUTOINCREMENT, item_code CHARACTER(6), treatment_of_code INTEGER)')
# Read drug list, prescriber type
with zipfile.ZipFile('2023-01-01-v3extracts.zip', 'r') as zipf:
with zipfile.ZipFile('data/2023-01-01-v3extracts.zip', 'r') as zipf:
# drug_xxx.txt
with zipf.open('drug_20230101.txt', 'r') as f:
df_drug = pd.read_csv(f, sep='!')
for _, drug in df_drug.iterrows():
# Skip already added
cur.execute('SELECT COUNT(*) FROM pbs_drug WHERE item_code=?', (drug['item-code'],))
if cur.fetchone()[0] > 0:
continue
cur.execute('INSERT INTO pbs_drug (item_code, mp_pt, tpuu_or_mpp_pt, restriction_flag, mq, repeats) VALUES (?, ?, ?, ?, ?, ?)', (drug['item-code'], drug['mp-pt'], drug['tpuu-or-mpp-pt'], drug['restriction-flag'], drug['mq'], drug['repeats']))
# Prescriber_type_xxx.txt
with zipf.open('Prescriber_type_20230101.txt', 'r') as f:
df_prescriber_type = pd.read_csv(f, sep='\t', header=0, names=['mp-pt', 'item-code', 'prescriber-type'])
df_drug = df_drug.merge(df_prescriber_type[['item-code', 'prescriber-type']], how='left', on='item-code')
for _, prescriber_type in df_prescriber_type.iterrows():
cur.execute('INSERT INTO pbs_prescriber_type (item_code, prescriber_type) VALUES (?, ?)', (prescriber_type['item-code'], prescriber_type['prescriber-type']))
# Filter only drugs able to be prescribed by medical practitioners
df_drug = df_drug[df_drug['prescriber-type'] == 'M']
# streamlined_xxx.txt (streamlined authorities)
for _, drug in df_drug[['item-code', 'mp-pt', 'tpuu-or-mpp-pt', 'restriction-flag', 'mq', 'repeats']].iterrows():
# Skip already added
cur.execute('SELECT COUNT(*) FROM pbs WHERE item_code=?', (drug['item-code'],))
if cur.fetchone()[0] > 0:
continue
# Add to SQL
cur.execute('INSERT INTO pbs (item_code, mp_pt, tpuu_or_mpp_pt, restriction_flag, mq, repeats) VALUES (?, ?, ?, ?, ?, ?)', (drug['item-code'], drug['mp-pt'], drug['tpuu-or-mpp-pt'], drug['restriction-flag'], drug['mq'], drug['repeats']))
# Read streamlined authorities
with zipfile.ZipFile('2023-01-01-v3extracts.zip', 'r') as zipf:
with zipf.open('streamlined_20230101.txt', 'r') as f:
df_streamlined = pd.read_csv(f, sep='\t')
df_streamlined = df_drug.merge(df_streamlined[['item-code', 'treatment-of-code']], how='inner', on='item-code')
for k, v in df_streamlined.groupby('item-code'):
cur.execute('UPDATE pbs SET streamlined_authorities=? WHERE item_code=?', (','.join(v['treatment-of-code'].astype(str)), k))
for _, streamlined in df_streamlined.iterrows():
cur.execute('INSERT INTO pbs_streamlined (item_code, treatment_of_code) VALUES (?, ?)', (streamlined['item-code'], streamlined['treatment-of-code']))
con.commit()