diff --git a/app/__init__.py b/app/__init__.py index 2fd3d82fefb5c8b25d7d0d93cae852662cfbc5c4..0a1725a407c7dbc81706fa4fbc72c83fc6c18147 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -15,7 +15,7 @@ from flask_cors import CORS, cross_origin app = Flask(__name__) CORS(app) -#CORS(app, resources={r"/*": {"cors_allowed_origins":"*"} } ) +# CORS(app, resources={r"/*": {"cors_allowed_origins":"*"} } ) #app.config['BABEL_DEFAULT_LOCALE'] = 'fin' #app.config['BABEL_TRANSLATION_DIRECTORIES'] ='C:/Users/Timo/git/pet-rating/app/translations' @@ -69,7 +69,8 @@ def get_locale(): # Run flask app with socketIO -socketio = SocketIO() +socketio = SocketIO(app, cors_allowed_origins="*") +# socketio = SocketIO() socketio.init_app(app) #mariabd mysql portti 3306 tarkista? diff --git a/app/experiment/templates/experiment_statistics.html b/app/experiment/templates/experiment_statistics.html index 817470eafe2cfc7e485c63e9c1c60db30827c24d..2dd5a34e0a711c2420e4845eca8d9a23cd95560e 100644 --- a/app/experiment/templates/experiment_statistics.html +++ b/app/experiment/templates/experiment_statistics.html @@ -36,7 +36,24 @@ <tr> <td>Number of finished ratings:</td> <td>{{ finished_ratings }} - <a class="btn btn-primary btn-info float-right" href="{{ url_for('download_csv', exp_id=exp.idexperiment) }}" role="button">Export results (csv)</a> + + + <button data-value="{{ exp.idexperiment }}" class="btn btn-primary float-right get-csv-results"> + + export results + + </button> + + <div id="export-link-container" class="hidden"> + <a id="export-link" class="float-right" href="{{ url_for('experiment.download_csv', exp_id=exp.idexperiment) }}" role="button">Download</a> + </div> + + + <div class="progress hidden"> + <div id="export-results-bar" class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" aria-valuenow="50" aria-valuemin="0" aria-valuemax="100" style="width: 0%"> + </div> + </div> + </td> </tr> </tbody> @@ -194,6 +211,8 @@ </table> <script src="{{ url_for('static', filename='lib/js/socket.io.js') }}" ></script> +<script src="{{ url_for('static', filename='js/urls.js') }}" ></script> <script src="{{ url_for('static', filename='js/getDrawing.js') }}" ></script> +<script src="{{ url_for('static', filename='js/getCSV.js') }}" ></script> {% endblock %} diff --git a/app/experiment/views.py b/app/experiment/views.py index d36ced6e557bd57e5740d377c89e8b3e93056ef0..462db81d83316335743ae20f58c8dbf2d0826663 100644 --- a/app/experiment/views.py +++ b/app/experiment/views.py @@ -1,12 +1,12 @@ -from flask_cors import CORS, cross_origin from app import socketio from flask_socketio import emit import embody_plot import os import secrets import json +from datetime import datetime, date from flask import ( Flask, @@ -40,8 +40,8 @@ from app.forms import ( EditQuestionForm, EditExperimentForm, UploadResearchBulletinForm, EditPageForm, RemoveExperimentForm, GenerateIdForm, CreateEmbodyForm ) -from app.utils import get_mean_from_slider_answers, map_answers_to_questions - +from app.utils import get_mean_from_slider_answers, map_answers_to_questions, \ + saved_data_as_file, timeit, generate_csv # Stimuli upload folder setting #APP_ROOT = os.path.dirname(os.path.abspath(__file__)) @@ -1032,13 +1032,9 @@ def create_embody(): @socketio.on('draw', namespace="/create_embody") -def create_embody(page_id): - - print("DRAW") - - page = page_id["page"] - embody = page_id["embody"] - +def create_embody(meta): + page = meta["page"] + embody = meta["embody"] img_path = embody_plot.get_coordinates(page, embody) app.logger.info(img_path) emit('end', {'path': img_path}) @@ -1046,8 +1042,64 @@ def create_embody(page_id): @socketio.on('end', namespace="/create_embody") def create_embody(): - print("connection end") emit('end', {'connection': 'off'}) -# EOF + + +@socketio.on('connect', namespace="/download_csv") +def create_embody(): + emit('success', {'connection': 'Start generating CSV file'}) + + + +from tempfile import mkstemp +from flask import send_file + +@socketio.on('generate_csv', namespace="/download_csv") +def create_embody(meta): + + exp_id = meta["exp_id"] + + csv = generate_csv(exp_id) + + if not csv: + emit('timeout') + + filename = "experiment_{}_{}".format( + exp_id, date.today().strftime("%Y-%m-%d")) + + fd, path = mkstemp() + + print(fd) + print(path) + + with os.fdopen(fd, 'w') as tmp: + tmp.write(csv) + tmp.flush() + + path = path.split('/')[-1] + + emit('file_ready', {'path': path, 'filename': filename}) + + # return saved_data_as_file(filename, csv) + + +@experiment_blueprint.route('/download_csv') +def download_csv(): + exp_id = request.args.get('exp_id', None) + path = request.args.get('path', None) + + filename = "experiment_{}_{}.csv".format( + exp_id, date.today().strftime("%Y-%m-%d")) + + path = '/tmp/' + path + + try: + return send_file(path, + mimetype='text/csv', + as_attachment=True, + attachment_filename=filename) + + finally: + os.remove(path) diff --git a/app/models.py b/app/models.py index ed4669cda1336d2d73e7b845897a8b5e7488489d..c792add29192ab35999a8898e477eff576fd505b 100644 --- a/app/models.py +++ b/app/models.py @@ -153,6 +153,12 @@ class answer (db.Model): answer = db.Column(db.String(120)) page_idpage = db.Column(db.Integer, db.ForeignKey('page.idpage')) + def question(self): + return int(self.question_idquestion) + + def result(self): + return int(self.answer) + def __repr__(self): return "<idanswer = '%s', question_idquestion = '%s', answer_set_idanswer_set = '%s', answer = '%s', page_idpage = '%s'>" % (self.idanswer, self.question_idquestion, self.answer_set_idanswer_set, self.answer, self.page_idpage) @@ -167,6 +173,12 @@ class embody_answer (db.Model): db.Integer, db.ForeignKey('embody_question.idembody')) coordinates = db.Column(db.Text) + def question(self): + return self.embody_question_idembody + + def result(self): + return self.coordinates + def __repr__(self): return "<idanswer = '%s', answer_set_idanswer_set = '%s', coordinates = '%s', page_idpage = '%s', embody_question_idembody='%s' >" % (self.idanswer, self.answer_set_idanswer_set, self.coordinates, self.page_idpage, self.embody_question_idembody) diff --git a/app/routes.py b/app/routes.py index cf6e2b4deb93873fa0ed5f06aa25ae483d84d444..34902a0d596e4ea088fc1bf6d7ad2e157653c8ce 100644 --- a/app/routes.py +++ b/app/routes.py @@ -21,7 +21,6 @@ from app.models import background_question_option from app.models import answer_set, answer, forced_id from app.models import user, trial_randomization from app.forms import LoginForm, RegisterForm, StartWithIdForm -from app.utils import saved_data_as_file, map_answers_to_questions, timeit # Stimuli upload folder setting APP_ROOT = os.path.dirname(os.path.abspath(__file__)) @@ -336,181 +335,7 @@ def view_research_notification(): return render_template('view_research_notification.html', research_notification_filename=research_notification_filename) -@timeit -@app.route('/download_csv') -@login_required -def download_csv(): - - exp_id = request.args.get('exp_id', None) - experiment_info = experiment.query.filter_by(idexperiment=exp_id).all() - - print(experiment_info) - - # answer sets with participant ids - participants = answer_set.query.filter_by( - experiment_idexperiment=exp_id).all() - - # pages aka stimulants - pages = page.query.filter_by(experiment_idexperiment=exp_id).all() - - # background questions - bg_questions = background_question.query.filter_by( - experiment_idexperiment=exp_id).all() - - # question - questions = question.query.filter_by(experiment_idexperiment=exp_id).all() - - # embody questions - embody_questions = embody_question.query.filter_by( - experiment_idexperiment=exp_id).all() - - csv = '' - - # create CSV-header - header = 'participant id;' - header += ';'.join([str(count) + '. bg_question: ' + q.background_question.strip() - for count, q in enumerate(bg_questions, 1)]) - - for idx in range(1, len(pages) + 1): - if len(questions) > 0: - header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. slider_question: ' + - question.question.strip() for count, question in enumerate(questions, 1)]) - - for idx in range(1, len(pages) + 1): - if len(embody_questions) > 0: - header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' + - question.picture.strip() for count, question in enumerate(embody_questions, 1)]) - - csv += header + '\r\n' - - csv += generate_answers(participants, pages, questions, embody_questions) - - filename = "experiment_{}_{}.csv".format( - exp_id, date.today().strftime("%Y-%m-%d")) - - return saved_data_as_file(filename, csv) - - -@timeit -def generate_answers(participants, pages, questions, embody_questions): - - csv = '' - answer_row = '' - - for participant in participants: - - - - # list only finished answer sets - if int(participant.answer_counter) == 0: - continue - - try: - # append user session id - answer_row += participant.session + ';' - - # append background question answers - bg_answers = background_question_answer.query.filter_by( - answer_set_idanswer_set=participant.idanswer_set).all() - bg_answers_list = [str(a.answer).strip() for a in bg_answers] - answer_row += ';'.join(bg_answers_list) + ';' - - # append slider answers - slider_answers = answer.query.filter_by( - answer_set_idanswer_set=participant.idanswer_set) \ - .order_by(answer.page_idpage, answer.question_idquestion) \ - .all() - - - pages_and_questions = {} - - for p in pages: - questions_list = [(p.idpage, a.idquestion) for a in questions] - pages_and_questions[p.idpage] = questions_list - - _questions = [ - item for sublist in pages_and_questions.values() for item in sublist] - - answers_list = map_answers_to_questions(slider_answers, _questions) - - # typecast elemnts to string - answers_list = [str(a).strip() for a in answers_list] - - answer_row += ';'.join(answers_list) + \ - ';' if slider_answers else len( - questions) * len(pages) * ';' - - # append embody answers (coordinates) - # save embody answers as bitmap images - embody_answers = embody_answer.query.filter_by( - answer_set_idanswer_set=participant.idanswer_set) \ - .order_by(embody_answer.page_idpage) \ - .all() - - pages_and_questions = {} - - for p in pages: - questions_list = [(p.idpage, a.idembody) for a in embody_questions] - pages_and_questions[p.idpage] = questions_list - - _questions = [ - item for sublist in pages_and_questions.values() for item in sublist] - - _embody_answers = map_answers_to_questions(embody_answers, _questions) - - answers_list = [] - - for answer_data in _embody_answers: - - if not answer_data: - answers_list.append('') - continue - - try: - coordinates = json.loads(answer_data.coordinates) - em_height = coordinates.get('height', 600) + 2 - em_width = coordinates.get('width', 200) + 2 - - coordinates_to_bitmap = [ - [0 for x in range(em_height)] for y in range(em_width)] - - coordinates = list( - zip(coordinates.get('x'), coordinates.get('y'))) - - for point in coordinates: - - try: - # for every brush stroke, increment the pixel - # value for every brush stroke - coordinates_to_bitmap[point[0]][point[1]] += 0.1 - except IndexError: - continue - - answers_list.append(json.dumps(coordinates_to_bitmap)) - - except ValueError as err: - app.logger(err) - - answer_row += ';'.join(answers_list) if embody_answers else \ - len(embody_questions) * len(pages) * ';' - - # old way to save only visited points: - # answers_list = [json.dumps( - # list(zip( json.loads(a.coordinates)['x'], - # json.loads(a.coordinates)['y']))) for a in embody_answers] - - except TypeError as err: - print(err) - - csv += answer_row + '\r\n' - answer_row = '' - return csv - - @app.route('/researcher_info') @login_required def researcher_info(): return render_template('researcher_info.html') - - -# EOF diff --git a/app/static/css/main.css b/app/static/css/main.css index eeb85710389fc8101bdd970f30d073288b47bf9d..72677599a838f26d30a18764052ab7499d5fd9f8 100644 --- a/app/static/css/main.css +++ b/app/static/css/main.css @@ -75,4 +75,10 @@ body { max-width: 90%; } -} \ No newline at end of file +} + + +#export-link-container { + margin-top: 20px; + padding: 10px; +} diff --git a/app/static/js/getDrawing.js b/app/static/js/getDrawing.js index 4504eb302e9aa17a0a4e9f5128036f3b2bd3ec82..9f349fbdff38d2d08857b87198d85cb45577f2af 100644 --- a/app/static/js/getDrawing.js +++ b/app/static/js/getDrawing.js @@ -1,9 +1,5 @@ -const baseURI = 'localhost/'; -//const baseURI = 'http://onni.utu.fi/'; -var getDrawingURI = baseURI + 'create_embody'; - $(document).ready(function()Â { var drawButtons = $(".embody-get-drawing"); @@ -23,7 +19,6 @@ $(document).ready(function()Â { }); socket.on('end', function(img) { - socket.disconnect() // Draw image to statistic -page diff --git a/app/utils.py b/app/utils.py index 694062d9383767cf89b38a908e2ce6013639e9b6..a36b220f3b2d936698a49cd7d040f5e434d7d1d3 100644 --- a/app/utils.py +++ b/app/utils.py @@ -2,10 +2,11 @@ import os import tempfile import time from itertools import zip_longest +import concurrent.futures from flask import send_file - - from app import app +from app.models import background_question, background_question_answer, \ + page, question, answer_set, answer, embody_answer, embody_question def timeit(method): @@ -17,7 +18,7 @@ def timeit(method): name = kw.get('log_name', method.__name__.upper()) kw['log_time'][name] = int((te - ts) * 1000) else: - print('{} {:2.2f} ms'.format(method.__name__, (te - ts) * 1000)) + app.logger.info('{} {:2.2f} ms'.format(method.__name__, (te - ts) * 1000)) return result return timed @@ -71,7 +72,7 @@ def get_values_from_list_of_answers(page_question, answers): def question_matches_answer(question, answer): - if (answer.page_idpage == question[0] and answer.question_idquestion == question[1]): + if (answer.page_idpage == question[0] and answer.question() == question[1]): return True return False @@ -98,7 +99,7 @@ def map_answers_to_questions(answers, questions): break if question_matches_answer(question, current_answer): - results[nth_question] = int(current_answer.answer) + results[nth_question] = current_answer.result nth_answer += 1 return results @@ -136,4 +137,163 @@ group by sub.answer_set_idanswer_set; # all possible page/question comobs select distinct p.idpage, q.idquestion from question q join page p on p.experiment_idexperiment=q.experiment_idexperiment where p.experiment_idexperiment = 2 order by p.idpage,q.idquestion; -''' \ No newline at end of file +''' + + +@timeit +def generate_csv(exp_id): + + # answer sets with participant ids + participants = answer_set.query.filter_by( + experiment_idexperiment=exp_id).all() + + # pages aka stimulants + pages = page.query.filter_by(experiment_idexperiment=exp_id).all() + + # background questions + bg_questions = background_question.query.filter_by( + experiment_idexperiment=exp_id).all() + + # question + questions = question.query.filter_by(experiment_idexperiment=exp_id).all() + + # embody questions + embody_questions = embody_question.query.filter_by( + experiment_idexperiment=exp_id).all() + + csv = '' + + # create CSV-header + header = 'participant id;' + header += ';'.join([str(count) + '. bg_question: ' + q.background_question.strip() + for count, q in enumerate(bg_questions, 1)]) + + for idx in range(1, len(pages) + 1): + if len(questions) > 0: + header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. slider_question: ' + + question.question.strip() for count, question in enumerate(questions, 1)]) + + for idx in range(1, len(pages) + 1): + if len(embody_questions) > 0: + header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' + + question.picture.strip() for count, question in enumerate(embody_questions, 1)]) + + csv += header + '\r\n' + + # filter empty answer_sets + participants = list(filter(lambda participant: True if int( + participant.answer_counter) > 0 else False, participants)) + + # We can use a with statement to ensure threads are cleaned up promptly + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + # Start the load operations and mark each future with its URL + future_to_answer = { + executor.submit(generate_answer_row, participant, pages, questions, embody_questions): participant + for participant in participants} + + for future in concurrent.futures.as_completed(future_to_answer): + # for testing purpose + # answer_row = future_to_answer[future] + try: + data = future.result() + csv += data + '\r\n' + except TimeoutError: + return None + except Exception as exc: + print('generated an exception: {}'.format(exc)) + + return csv + + +def generate_answer_row(participant, pages, questions, embody_questions): + # TODO: refactor + + answer_row = '' + + # append user session id + answer_row += participant.session + ';' + + # append background question answers + bg_answers = background_question_answer.query.filter_by( + answer_set_idanswer_set=participant.idanswer_set).all() + bg_answers_list = [str(a.answer).strip() for a in bg_answers] + answer_row += ';'.join(bg_answers_list) + ';' + + # append slider answers + slider_answers = answer.query.filter_by( + answer_set_idanswer_set=participant.idanswer_set) \ + .order_by(answer.page_idpage, answer.question_idquestion) \ + .all() + + pages_and_questions = {} + + for p in pages: + questions_list = [(p.idpage, a.idquestion) for a in questions] + pages_and_questions[p.idpage] = questions_list + + _questions = [ + item for sublist in pages_and_questions.values() for item in sublist] + + answers_list = map_answers_to_questions(slider_answers, _questions) + + # typecast elemnts to string + answers_list = [str(a).strip() for a in answers_list] + + answer_row += ';'.join(answers_list) + \ + ';' if slider_answers else len( + questions) * len(pages) * ';' + + # append embody answers (coordinates) + # save embody answers as bitmap images + embody_answers = embody_answer.query.filter_by( + answer_set_idanswer_set=participant.idanswer_set) \ + .order_by(embody_answer.page_idpage) \ + .all() + + pages_and_questions = {} + + for p in pages: + questions_list = [(p.idpage, a.idembody) for a in embody_questions] + pages_and_questions[p.idpage] = questions_list + + _questions = [ + item for sublist in pages_and_questions.values() for item in sublist] + + _embody_answers = map_answers_to_questions(embody_answers, _questions) + + answers_list = [] + + for answer_data in _embody_answers: + if not answer_data: + answers_list.append('') + continue + + try: + coordinates = json.loads(answer_data) + em_height = coordinates.get('height', 600) + 2 + em_width = coordinates.get('width', 200) + 2 + + coordinates_to_bitmap = [ + [0 for x in range(em_height)] for y in range(em_width)] + + coordinates = list( + zip(coordinates.get('x'), coordinates.get('y'))) + + for point in coordinates: + + try: + # for every brush stroke, increment the pixel + # value for every brush stroke + coordinates_to_bitmap[point[0]][point[1]] += 0.1 + except IndexError: + continue + + answers_list.append(json.dumps(coordinates_to_bitmap)) + + except ValueError as err: + app.logger(err) + + answer_row += ';'.join(answers_list) if embody_answers else \ + len(embody_questions) * len(pages) * ';' + + return answer_row diff --git a/config.py b/config.py index 087604733d53112a8fd65554e3163312f828cde4..eced748970d173cace8a0c38315494b997777065 100644 --- a/config.py +++ b/config.py @@ -1,13 +1,12 @@ +from decouple import config import os basedir = os.path.abspath(os.path.dirname(__file__)) -from decouple import config class Config(object): - #seret key is set in __ini__.py - #SECRET_KEY = os.environ.get('SECRET_KEY') or 'you-will-never-guess' - + # seret key is set in __ini__.py + #SECRET_KEY = os.environ.get('SECRET_KEY') or 'you-will-never-guess' LANGUAGES = ['en', 'fi', 'fa', 'el', 'it', 'zh'] @@ -19,23 +18,25 @@ class Config(object): SQLALCHEMY_TRACK_MODIFICATIONS = False """ - #MariaDB mysql database settings + # MariaDB mysql database settings MYSQL_USER = config('MYSQL_USER') MYSQL_PASSWORD = config('MYSQL_PASSWORD') MYSQL_SERVER = config('MYSQL_SERVER') MYSQL_DB = config('MYSQL_DB') - - SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://'+MYSQL_USER+':'+MYSQL_PASSWORD+'@'+MYSQL_SERVER+'/'+MYSQL_DB+'?charset=utf8mb4' - + + SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://'+MYSQL_USER+':' + \ + MYSQL_PASSWORD+'@'+MYSQL_SERVER+'/'+MYSQL_DB+'?charset=utf8mb4' + SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_ENGINE_OPTIONS = { - "pool_pre_ping": True, - "pool_recycle": 300, - "max_overflow": 30, - "pool_size": 20 + "pool_pre_ping": True, + "pool_recycle": 300, + "max_overflow": 30, + "pool_size": 20 } - + TEMPLATES_AUTO_RELOAD = True - DEBUG = False + + DEBUG = True diff --git a/embody_plot.py b/embody_plot.py index 0d61fad96a73dc85a9245318d83f3cfd639edee1..cd74f0b8a84e6deab8a57eaab6073a2c4a896955 100644 --- a/embody_plot.py +++ b/embody_plot.py @@ -234,7 +234,7 @@ def plot_coordinates(coordinates, image_path=DEFAULT_IMAGE_PATH): ax1.plot(coordinates["x"],coordinates["y"], 'ro', alpha=0.2) ax1.imshow(image, alpha=0.6) - app.logger.info("iamge plotted") + app.logger.info("image plotted") # return figure for saving/etc... return fig diff --git a/requirements.txt b/requirements.txt index dc5de6c46671b624a8d5237027838bd05175ab99..c44df47212d514b6e1462704d61863c0561e066b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,8 @@ Flask-Cors==3.0.7 Flask-Login==0.4.1 Flask-Migrate==2.2.1 Flask-Session==0.3.1 -Flask-SocketIO==3.3.2 +Flask-SocketIO==4.3.0 +# Flask-SocketIO==3.3.2 Flask-SQLAlchemy==2.3.2 Flask-Uploads==0.2.1 Flask-WTF==0.14.2 @@ -40,8 +41,10 @@ PyMySQL==0.9.3 pyparsing==2.3.1 python-dateutil==2.7.3 python-editor==1.0.3 -python-engineio==3.5.1 -python-socketio==3.1.2 +python-engineio==3.13.0 +# python-engineio==3.5.1 +# python-socketio==3.1.2 +python-socketio==4.6.0 pytz==2018.7 rope==0.12.0 scipy==1.2.1 @@ -54,4 +57,4 @@ visitor==0.1.3 Werkzeug==0.14.1 WTForms==2.2.1 WTForms-SQLAlchemy==0.1 -python-decouple \ No newline at end of file +python-decouple