From 788c5c006cc241967ad148bb41a8e088ba06284c Mon Sep 17 00:00:00 2001 From: RunasSudo Date: Tue, 12 Dec 2017 20:02:02 +1030 Subject: [PATCH] Smush the WorkflowTasks and Tasks together; liberally apply superglue and duct tape --- eos/base/workflow.py | 21 ++++- eos/core/objects/__init__.py | 39 ++++++++-- eos/core/tasks/__init__.py | 78 ++++++++++++++----- eos/core/tasks/direct.py | 24 +++++- eos/core/tests.py | 1 + eos/js.py | 3 + eos/psr/election.py | 2 - eos/psr/workflow.py | 2 +- eosweb/core/main.py | 16 +++- eosweb/core/static/css/main.css | 5 ++ eosweb/core/templates/active_tasks_menu.html | 42 ++++++++++ eosweb/core/templates/base.html | 3 + .../core/templates/election/admin/admin.html | 8 +- local_settings.example.py | 2 + 14 files changed, 204 insertions(+), 42 deletions(-) create mode 100644 eosweb/core/templates/active_tasks_menu.html diff --git a/eos/base/workflow.py b/eos/base/workflow.py index 620fada..d0b5ada 100644 --- a/eos/base/workflow.py +++ b/eos/base/workflow.py @@ -15,6 +15,7 @@ # along with this program. If not, see . from eos.core.objects import * +from eos.core.tasks import * class WorkflowTask(EmbeddedObject): class Status: @@ -64,7 +65,7 @@ class WorkflowTask(EmbeddedObject): @classmethod def satisfies(cls, descriptor): - return cls._name == descriptor or descriptor in cls.provides or (descriptor in EosObject.objects and issubclass(cls, EosObject.objects[descriptor])) + return cls._name == descriptor or descriptor in cls.provides or (descriptor in EosObject.objects and issubclass(cls, EosObject.lookup(descriptor))) def on_enter(self): self.exit() @@ -113,6 +114,22 @@ class Workflow(EmbeddedObject): except StopIteration: return None +class WorkflowTaskEntryTask(Task): + election_id = UUIDField() + workflow_task = StringField() + + def _run(self): + election = EosObject.lookup('eos.base.election.Election').get_by_id(self.election_id) + task = election.workflow.get_task(self.workflow_task) + task.enter() + election.save() + + @property + def label(self): + election = EosObject.lookup('eos.base.election.Election').get_by_id(self.election_id) + task = election.workflow.get_task(self.workflow_task) + return task.label + ' – ' + election.name + # Concrete tasks # ============== @@ -138,7 +155,7 @@ class TaskDecryptVotes(WorkflowTask): election = self.recurse_parents('eos.base.election.Election') for _ in range(len(election.questions)): - election.results.append(EosObject.objects['eos.base.election.RawResult']()) + election.results.append(EosObject.lookup('eos.base.election.RawResult')()) for voter in election.voters: if len(voter.votes) > 0: diff --git a/eos/core/objects/__init__.py b/eos/core/objects/__init__.py index 3473170..ae0b8d3 100644 --- a/eos/core/objects/__init__.py +++ b/eos/core/objects/__init__.py @@ -37,6 +37,7 @@ if is_python: import base64 from datetime import datetime import hashlib + import importlib import json import uuid __pragma__('noskip') @@ -177,9 +178,6 @@ class EosObjectType(type): cls._name = (cls.__module__ + '.' + cls.__name__).replace('.js.', '.').replace('.python.', '.') #TNYI: qualname if name != 'EosObject': EosObject.objects[cls._name] = cls - if '_db_name' not in attrs: - # Don't inherit _db_name, use only if explicitly given - cls._db_name = cls._name return cls class EosObject(metaclass=EosObjectType): @@ -195,7 +193,7 @@ class EosObject(metaclass=EosObjectType): def recurse_parents(self, cls): #if not isinstance(cls, type): if isinstance(cls, str): - cls = EosObject.objects[cls] + cls = EosObject.lookup(cls) if isinstance(self, cls): return self @@ -208,6 +206,13 @@ class EosObject(metaclass=EosObjectType): return False return EosObject.serialise_and_wrap(self) == EosObject.serialise_and_wrap(other) + @staticmethod + def lookup(name): + if name in EosObject.objects: + return EosObject.objects[name] + importlib.import_module(name[:name.rindex('.')]) + return EosObject.objects[name] + @staticmethod def serialise_and_wrap(value, object_type=None, for_hash=False, should_protect=False): if object_type: @@ -220,7 +225,9 @@ class EosObject(metaclass=EosObjectType): def deserialise_and_unwrap(value, object_type=None): if object_type: return object_type.deserialise(value) - return EosObject.objects[value['type']].deserialise(value['value']) + if value: + return EosObject.lookup(value['type']).deserialise(value['value']) + return None @staticmethod def to_json(value): @@ -399,7 +406,25 @@ class DocumentObject(EosObject, metaclass=DocumentObjectType): attrs[val.internal_name] = val.deserialise(value[val.real_name]) return cls(**attrs) -class TopLevelObject(DocumentObject): +class TopLevelObjectType(DocumentObjectType): + def __new__(meta, name, bases, attrs): + cls = DocumentObjectType.__new__(meta, name, bases, attrs) + + # TopLevelObject obviously has no _db_name + if cls._name == 'eos.core.objects.TopLevelObject': + pass + else: + if '_db_name' not in attrs: + cls._db_name = cls._name + # If _db_name is False, then explicitly use the _name. Otherwise, inherit. + if cls._db_name is not False: + for base in bases: + if hasattr(base, '_db_name'): + cls._db_name = base._db_name + break + return cls + +class TopLevelObject(DocumentObject, metaclass=TopLevelObjectType): def save(self): #res = db[self._name].replace_one({'_id': self.serialise()['_id']}, self.serialise(), upsert=True) #res = dbinfo.db[self._db_name].replace_one({'_id': self._fields['_id'].serialise(self._id)}, EosObject.serialise_and_wrap(self), upsert=True) @@ -411,6 +436,8 @@ class TopLevelObject(DocumentObject): @classmethod def get_by_id(cls, _id): + if not isinstance(_id, str): + _id = str(_id) return EosObject.deserialise_and_unwrap(dbinfo.provider.get_by_id(cls._db_name, _id)) class EmbeddedObject(DocumentObject): diff --git a/eos/core/tasks/__init__.py b/eos/core/tasks/__init__.py index a8c5522..eadf612 100644 --- a/eos/core/tasks/__init__.py +++ b/eos/core/tasks/__init__.py @@ -28,8 +28,14 @@ class Task(TopLevelObject): TIMEOUT = -20 _id = UUIDField() - status = IntField(default=0) run_strategy = EmbeddedObjectField() + + run_at = DateTimeField() + + started_at = DateTimeField() + completed_at = DateTimeField() + + status = IntField(default=0) messages = ListField(StringField()) def run(self): @@ -38,26 +44,60 @@ class Task(TopLevelObject): def _run(self): pass +class DummyTask(Task): + _db_name = Task._db_name + label = 'A dummy task' + + def _run(self): + if is_python: + #__pragma__('skip') + import time + #__pragma__('noskip') + time.sleep(15) + class RunStrategy(DocumentObject): def run(self, task): raise Exception('Not implemented') -class DirectRunStrategy(RunStrategy): - def run(self, task): - task.status = Task.Status.PROCESSING - task.save() +class TaskScheduler: + @staticmethod + def pending_tasks(): + pending_tasks = [] + tasks = Task.get_all() - try: - task._run() - task.status = Task.Status.COMPLETE - task.save() - except Exception as e: - task.status = Task.Status.FAILED - if is_python: - #__pragma__('skip') - import traceback - #__pragma__('noskip') - task.messages.append(traceback.format_exc()) - else: - task.messages.append(repr(e)) - task.save() + for task in tasks: + if task.status == Task.Status.READY and task.run_at and task.run_at < DateTimeField.now(): + pending_tasks.append(task) + + return pending_tasks + + @staticmethod + def active_tasks(): + active_tasks = [] + tasks = Task.get_all() + + for task in tasks: + if task.status == Task.Status.PROCESSING: + active_tasks.append(task) + + return active_tasks + + @staticmethod + def completed_tasks(limit=None): + completed_tasks = [] + tasks = Task.get_all() + + for task in tasks: + if task.status == Task.Status.COMPLETE or task.status < 0: + completed_tasks.append(task) + + if limit: + completed_tasks.sort(key=lambda x: x.completed_at) + completed_tasks = completed_tasks[-limit:] + + return completed_tasks + + @staticmethod + def tick(): + for task in TaskScheduler.pending_tasks(): + task.run() diff --git a/eos/core/tasks/direct.py b/eos/core/tasks/direct.py index f5138fa..d0e883a 100644 --- a/eos/core/tasks/direct.py +++ b/eos/core/tasks/direct.py @@ -15,5 +15,27 @@ # along with this program. If not, see . from eos.core.tasks import * +from eos.core.objects import * - +class DirectRunStrategy(RunStrategy): + def run(self, task): + task.status = Task.Status.PROCESSING + task.started_at = DateTimeField.now() + task.save() + + try: + task._run() + task.status = Task.Status.COMPLETE + task.completed_at = DateTimeField.now() + task.save() + except Exception as e: + task.status = Task.Status.FAILED + task.completed_at = DateTimeField.now() + if is_python: + #__pragma__('skip') + import traceback + #__pragma__('noskip') + task.messages.append(traceback.format_exc()) + else: + task.messages.append(repr(e)) + task.save() diff --git a/eos/core/tests.py b/eos/core/tests.py index 81c6957..475eedd 100644 --- a/eos/core/tests.py +++ b/eos/core/tests.py @@ -18,6 +18,7 @@ from eos.core.bigint import * from eos.core.objects import * from eos.core.hashing import * from eos.core.tasks import * +from eos.core.tasks.direct import * # Common library things # =================== diff --git a/eos/js.py b/eos/js.py index 56608db..ab1bfa5 100644 --- a/eos/js.py +++ b/eos/js.py @@ -17,6 +17,9 @@ import eos.core.objects import eos.core.bigint import eos.core.hashing +import eos.core.tests +import eos.core.tasks +import eos.core.tasks.direct import eos.base.election import eos.base.workflow diff --git a/eos/psr/election.py b/eos/psr/election.py index c23e207..0e859ec 100644 --- a/eos/psr/election.py +++ b/eos/psr/election.py @@ -225,8 +225,6 @@ class InternalMixingTrustee(MixingTrustee): return True class PSRElection(Election): - _db_name = Election._name - sk = EmbeddedObjectField(SEGPrivateKey, is_protected=True) # TODO: Threshold public_key = EmbeddedObjectField(SEGPublicKey) diff --git a/eos/psr/workflow.py b/eos/psr/workflow.py index 52d67b0..ac8a69d 100644 --- a/eos/psr/workflow.py +++ b/eos/psr/workflow.py @@ -66,7 +66,7 @@ class TaskDecryptVotes(eos.base.workflow.TaskDecryptVotes): election = self.recurse_parents('eos.base.election.Election') for _ in range(len(election.questions)): - election.results.append(EosObject.objects['eos.base.election.RawResult']()) + election.results.append(EosObject.lookup('eos.base.election.RawResult')()) for i in range(len(election.mixing_trustees[-1].mixed_questions)): for encrypted_answer in election.mixing_trustees[-1].mixed_questions[i]: diff --git a/eosweb/core/main.py b/eosweb/core/main.py index 85685ab..380f608 100644 --- a/eosweb/core/main.py +++ b/eosweb/core/main.py @@ -18,7 +18,9 @@ import click import flask from eos.core.objects import * +from eos.core.tasks import * from eos.base.election import * +from eos.base.workflow import * from eos.psr.crypto import * from eos.psr.election import * from eos.psr.mixnet import * @@ -152,6 +154,12 @@ def verify_election(electionid): def inject_globals(): return {'eos': eos, 'eosweb': eosweb, 'SHA256': eos.core.hashing.SHA256} +# Tickle the plumbus every request +@app.before_request +def tick_scheduler(): + # Process pending tasks + TaskScheduler.tick() + # === Views === @app.route('/') @@ -217,12 +225,12 @@ def election_admin_summary(election): @using_election @election_admin def election_admin_enter_task(election): - task = election.workflow.get_task(flask.request.args['task_name']) - if task.status != WorkflowTask.Status.READY: + workflow_task = election.workflow.get_task(flask.request.args['task_name']) + if workflow_task.status != WorkflowTask.Status.READY: return flask.Response('Task is not yet ready or has already exited', 409) - task.enter() - election.save() + task = WorkflowTaskEntryTask(election_id=election._id, workflow_task=workflow_task._name, status=Task.Status.READY, run_strategy=EosObject.lookup(app.config['TASK_RUN_STRATEGY'])()) + task.run() return flask.redirect(flask.url_for('election_admin_summary', election_id=election._id)) diff --git a/eosweb/core/static/css/main.css b/eosweb/core/static/css/main.css index 6bc38b3..53b39f2 100644 --- a/eosweb/core/static/css/main.css +++ b/eosweb/core/static/css/main.css @@ -40,6 +40,11 @@ margin-left: -1.75rem; } +/* Fix display of multiple right-aligned menu items */ +.ui.menu:not(.vertical) .right.item ~ .right.item { + margin-left: 0 !important; +} + @media print { body, html { /* Default height: 100% causes blank pages */ diff --git a/eosweb/core/templates/active_tasks_menu.html b/eosweb/core/templates/active_tasks_menu.html new file mode 100644 index 0000000..2240172 --- /dev/null +++ b/eosweb/core/templates/active_tasks_menu.html @@ -0,0 +1,42 @@ +{# + Eos - Verifiable elections + Copyright © 2017 RunasSudo (Yingtong Li) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . +#} + + diff --git a/eosweb/core/templates/base.html b/eosweb/core/templates/base.html index 4767ca6..0ee6d7e 100644 --- a/eosweb/core/templates/base.html +++ b/eosweb/core/templates/base.html @@ -28,6 +28,9 @@ Eos Voting for {{ eosweb.app.config['ORG_NAME'] }} Source Code {% if session.user %} + {% if session.user.is_admin() %} + {% include 'active_tasks_menu.html' %} + {% endif %}
{{ session.user.name }}
diff --git a/eosweb/core/templates/election/admin/admin.html b/eosweb/core/templates/election/admin/admin.html index 30031cc..1c656a1 100644 --- a/eosweb/core/templates/election/admin/admin.html +++ b/eosweb/core/templates/election/admin/admin.html @@ -24,14 +24,8 @@
    {% for task in election.workflow.tasks %} {% if task.status == eos.base.workflow.WorkflowTask.Status.READY %} -
  • {{ task.label }}
    • +
  • {{ task.label }}
    • {% endif %} {% endfor %}
- - {% endblock %} diff --git a/local_settings.example.py b/local_settings.example.py index ca354be..afcbacd 100644 --- a/local_settings.example.py +++ b/local_settings.example.py @@ -14,6 +14,8 @@ ADMINS = [ #eos.redditauth.election.RedditUser(username='xxxxxxxx') ] +TASK_RUN_STRATEGY = 'eos.core.tasks.direct.DirectRunStrategy' + # MongoDB DB_TYPE = 'mongodb'