There will be a short maintenance break on Wed 27.10. at 12:00. Estimated time 30 minutes.

Commit 108ee594 authored by Ossi Laine's avatar Ossi Laine
Browse files

Merge branch 'feat-optimize-downloading-dataset' into 'dev'

Feat optimize downloading dataset

See merge request tithei/pet-rating!4
parents 2c7fa1c8 912429a3
...@@ -15,7 +15,7 @@ from flask_cors import CORS, cross_origin ...@@ -15,7 +15,7 @@ from flask_cors import CORS, cross_origin
app = Flask(__name__) app = Flask(__name__)
CORS(app) CORS(app)
#CORS(app, resources={r"/*": {"cors_allowed_origins":"*"} } ) # CORS(app, resources={r"/*": {"cors_allowed_origins":"*"} } )
#app.config['BABEL_DEFAULT_LOCALE'] = 'fin' #app.config['BABEL_DEFAULT_LOCALE'] = 'fin'
#app.config['BABEL_TRANSLATION_DIRECTORIES'] ='C:/Users/Timo/git/pet-rating/app/translations' #app.config['BABEL_TRANSLATION_DIRECTORIES'] ='C:/Users/Timo/git/pet-rating/app/translations'
...@@ -68,9 +68,6 @@ def get_locale(): ...@@ -68,9 +68,6 @@ def get_locale():
""" """
# Run flask app with socketIO
socketio = SocketIO()
socketio.init_app(app)
#mariabd mysql portti 3306 tarkista? #mariabd mysql portti 3306 tarkista?
...@@ -81,6 +78,11 @@ migrate = Migrate(app, db) ...@@ -81,6 +78,11 @@ migrate = Migrate(app, db)
login = LoginManager(app) login = LoginManager(app)
login.login_view = 'login' login.login_view = 'login'
# Run flask app with socketIO
socketio = SocketIO(app, cors_allowed_origins="*")
# socketio = SocketIO()
socketio.init_app(app)
# Register blueprints # Register blueprints
from .task.views import task_blueprint from .task.views import task_blueprint
from .experiment.views import experiment_blueprint from .experiment.views import experiment_blueprint
......
...@@ -36,7 +36,25 @@ ...@@ -36,7 +36,25 @@
<tr> <tr>
<td>Number of finished ratings:</td> <td>Number of finished ratings:</td>
<td>{{ finished_ratings }} <td>{{ finished_ratings }}
<a class="btn btn-primary btn-info float-right" href="{{ url_for('download_csv', exp_id=exp.idexperiment) }}" role="button">Export results (csv)</a>
<button data-value="{{ exp.idexperiment }}" class="btn btn-primary float-right get-csv-results">
export results
</button>
<div id="export-link-container" class="hidden">
<a id="export-link" class="float-right" href="{{ url_for('experiment.download_csv', exp_id=exp.idexperiment) }}" role="button"></a>
<p id="export-error"></p>
</div>
<div class="progress hidden">
<div id="export-results-bar" class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" aria-valuenow="50" aria-valuemin="0" aria-valuemax="100" style="width: 0%">
</div>
</div>
</td> </td>
</tr> </tr>
</tbody> </tbody>
...@@ -194,6 +212,8 @@ ...@@ -194,6 +212,8 @@
</table> </table>
<script src="{{ url_for('static', filename='lib/js/socket.io.js') }}" ></script> <script src="{{ url_for('static', filename='lib/js/socket.io.js') }}" ></script>
<script src="{{ url_for('static', filename='js/urls.js') }}" ></script>
<script src="{{ url_for('static', filename='js/getDrawing.js') }}" ></script> <script src="{{ url_for('static', filename='js/getDrawing.js') }}" ></script>
<script src="{{ url_for('static', filename='js/getCSV.js') }}" ></script>
{% endblock %} {% endblock %}
from flask_cors import CORS, cross_origin
from app import socketio
from flask_socketio import emit
import embody_plot
import os import os
import secrets import secrets
import json from datetime import date
from tempfile import mkstemp
from flask_socketio import emit
from sqlalchemy import and_
from flask_login import login_required
from werkzeug import secure_filename
from flask import ( from flask import (
Flask,
render_template, render_template,
request, request,
session,
flash, flash,
redirect, redirect,
url_for, url_for,
Blueprint, Blueprint,
jsonify send_file
) )
from wtforms import Form from app import app, db, socketio
from sqlalchemy import and_, update
from flask_login import login_required
from werkzeug import secure_filename
from app import app, db
from app.routes import APP_ROOT from app.routes import APP_ROOT
from app.models import background_question, experiment from app.models import background_question, experiment
from app.models import background_question_answer from app.models import background_question_answer
from app.models import page, question from app.models import page, question
from app.models import background_question_option from app.models import background_question_option
from app.models import answer_set, answer, forced_id from app.models import answer_set, answer, forced_id
from app.models import user, trial_randomization from app.models import trial_randomization
from app.models import embody_answer, embody_question from app.models import embody_answer, embody_question
from app.forms import ( from app.forms import (
CreateBackgroundQuestionForm, CreateBackgroundQuestionForm,
...@@ -40,8 +33,10 @@ from app.forms import ( ...@@ -40,8 +33,10 @@ from app.forms import (
EditQuestionForm, EditExperimentForm, UploadResearchBulletinForm, EditQuestionForm, EditExperimentForm, UploadResearchBulletinForm,
EditPageForm, RemoveExperimentForm, GenerateIdForm, CreateEmbodyForm EditPageForm, RemoveExperimentForm, GenerateIdForm, CreateEmbodyForm
) )
from app.utils import get_mean_from_slider_answers, map_answers_to_questions from app.utils import get_mean_from_slider_answers, map_answers_to_questions, \
generate_csv
import embody_plot
# Stimuli upload folder setting # Stimuli upload folder setting
#APP_ROOT = os.path.dirname(os.path.abspath(__file__)) #APP_ROOT = os.path.dirname(os.path.abspath(__file__))
...@@ -957,6 +952,8 @@ def statistics(): ...@@ -957,6 +952,8 @@ def statistics():
questions = question.query.filter_by(experiment_idexperiment=exp_id).all() questions = question.query.filter_by(experiment_idexperiment=exp_id).all()
pages_and_questions = {} pages_and_questions = {}
'''
for p in pages: for p in pages:
questions_list = [(p.idpage, a.idquestion) for a in questions] questions_list = [(p.idpage, a.idquestion) for a in questions]
pages_and_questions[p.idpage] = questions_list pages_and_questions[p.idpage] = questions_list
...@@ -965,27 +962,34 @@ def statistics(): ...@@ -965,27 +962,34 @@ def statistics():
# those are in answer table as page_idpage and question_idquestion respectively # those are in answer table as page_idpage and question_idquestion respectively
slider_answers = {} slider_answers = {}
for participant in participants: for participant in participants:
if participant.answer_counter > 0:
answers = answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set)\
.order_by(answer.page_idpage)\
.all()
# flatten pages and questions to list of tuples (page_id, question_id) if int(participant.answer_counter) == 0:
_questions = [ continue
item for sublist in pages_and_questions.values() for item in sublist]
answers = answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set)\
.order_by(answer.page_idpage)\
.all()
slider_answers[participant.session] = map_answers_to_questions( # flatten pages and questions to list of tuples (page_id, question_id)
answers, _questions) _questions = [
item for sublist in pages_and_questions.values() for item in sublist]
slider_answers[participant.session] = map_answers_to_questions(
answers, _questions)
mean = get_mean_from_slider_answers(slider_answers) mean = get_mean_from_slider_answers(slider_answers)
# slider_answers['mean'] = get_mean_from_slider_answers(slider_answers) # slider_answers['mean'] = get_mean_from_slider_answers(slider_answers)
slider_answers = { slider_answers = {
'mean': mean 'mean': mean
} }
'''
slider_answers = {}
# Background question answers # Background question answers
bg_questions = background_question.query.filter_by( bg_questions = background_question.query.filter_by(
experiment_idexperiment=exp_id).all() experiment_idexperiment=exp_id).all()
...@@ -1012,8 +1016,27 @@ def statistics(): ...@@ -1012,8 +1016,27 @@ def statistics():
finished_ratings=finished_ratings, finished_ratings=finished_ratings,
question_headers=question_headers, question_headers=question_headers,
stimulus_headers=stimulus_headers, stimulus_headers=stimulus_headers,
embody_questions=embody_questions embody_questions=embody_questions)
)
@experiment_blueprint.route('/download_csv')
def download_csv():
exp_id = request.args.get('exp_id', None)
path = request.args.get('path', None)
filename = "experiment_{}_{}.csv".format(
exp_id, date.today().strftime("%Y-%m-%d"))
path = '/tmp/' + path
try:
return send_file(path,
mimetype='text/csv',
as_attachment=True,
attachment_filename=filename)
finally:
os.remove(path)
def remove_rows(rows): def remove_rows(rows):
...@@ -1024,27 +1047,55 @@ def remove_rows(rows): ...@@ -1024,27 +1047,55 @@ def remove_rows(rows):
@socketio.on('connect', namespace="/create_embody") @socketio.on('connect', namespace="/create_embody")
def create_embody(): def start_create_embody():
emit('success', {'connection': 'on'}) emit('success', {'connection': 'on'})
@socketio.on('draw', namespace="/create_embody") @socketio.on('draw', namespace="/create_embody")
def create_embody(page_id): def create_embody(meta):
page = meta["page"]
print("DRAW") embody = meta["embody"]
page = page_id["page"]
embody = page_id["embody"]
img_path = embody_plot.get_coordinates(page, embody) img_path = embody_plot.get_coordinates(page, embody)
app.logger.info(img_path) app.logger.info(img_path)
emit('end', {'path': img_path}) emit('end', {'path': img_path})
@socketio.on('end', namespace="/create_embody") @socketio.on('connect', namespace="/download_csv")
def create_embody(): def start_download_csv():
print("connection end") emit('success', {'connection': 'Start generating CSV file'})
emit('end', {'connection': 'off'})
# EOF @socketio.on('generate_csv', namespace="/download_csv")
def download_csv(meta):
exp_id = meta["exp_id"]
data = generate_csv(exp_id)
# error handling
if isinstance(data, Exception):
emit('timeout', {'exc': str(data)})
return
# create temporary file
fd, path = mkstemp()
with os.fdopen(fd, 'w') as tmp:
tmp.write(data)
tmp.flush()
# return path and filename to front so user can start downloading
filename = "experiment_{}_{}".format(
exp_id, date.today().strftime("%Y-%m-%d"))
path = path.split('/')[-1]
emit('file_ready', {'path': path, 'filename': filename})
@socketio.on('end', namespace="/download_csv")
def end_download_csv():
# TODO: not working solution... db session keeps hanging after socket session has ended
# mysqld timeout is set to 180s, so it kills hanging connections, but this is not a good solution
db.session.close()
@socketio.on('end', namespace="/create_embody")
def end_create_embody():
db.session.close()
...@@ -153,6 +153,12 @@ class answer (db.Model): ...@@ -153,6 +153,12 @@ class answer (db.Model):
answer = db.Column(db.String(120)) answer = db.Column(db.String(120))
page_idpage = db.Column(db.Integer, db.ForeignKey('page.idpage')) page_idpage = db.Column(db.Integer, db.ForeignKey('page.idpage'))
def question(self):
return int(self.question_idquestion)
def result(self):
return int(self.answer)
def __repr__(self): def __repr__(self):
return "<idanswer = '%s', question_idquestion = '%s', answer_set_idanswer_set = '%s', answer = '%s', page_idpage = '%s'>" % (self.idanswer, self.question_idquestion, self.answer_set_idanswer_set, self.answer, self.page_idpage) return "<idanswer = '%s', question_idquestion = '%s', answer_set_idanswer_set = '%s', answer = '%s', page_idpage = '%s'>" % (self.idanswer, self.question_idquestion, self.answer_set_idanswer_set, self.answer, self.page_idpage)
...@@ -167,6 +173,12 @@ class embody_answer (db.Model): ...@@ -167,6 +173,12 @@ class embody_answer (db.Model):
db.Integer, db.ForeignKey('embody_question.idembody')) db.Integer, db.ForeignKey('embody_question.idembody'))
coordinates = db.Column(db.Text) coordinates = db.Column(db.Text)
def question(self):
return self.embody_question_idembody
def result(self):
return self.coordinates
def __repr__(self): def __repr__(self):
return "<idanswer = '%s', answer_set_idanswer_set = '%s', coordinates = '%s', page_idpage = '%s', embody_question_idembody='%s' >" % (self.idanswer, self.answer_set_idanswer_set, self.coordinates, self.page_idpage, self.embody_question_idembody) return "<idanswer = '%s', answer_set_idanswer_set = '%s', coordinates = '%s', page_idpage = '%s', embody_question_idembody='%s' >" % (self.idanswer, self.answer_set_idanswer_set, self.coordinates, self.page_idpage, self.embody_question_idembody)
......
import os import os
import random import random
import secrets import secrets
from datetime import datetime, date from datetime import datetime
import json
from flask import (render_template, from flask import (render_template,
request, request,
...@@ -16,12 +15,11 @@ from flask_login import current_user, login_user, logout_user, login_required ...@@ -16,12 +15,11 @@ from flask_login import current_user, login_user, logout_user, login_required
from app import app, db from app import app, db
from app.models import background_question, experiment from app.models import background_question, experiment
from app.models import background_question_answer from app.models import background_question_answer
from app.models import page, question, embody_question, embody_answer from app.models import page
from app.models import background_question_option from app.models import background_question_option
from app.models import answer_set, answer, forced_id from app.models import answer_set, forced_id
from app.models import user, trial_randomization from app.models import user, trial_randomization
from app.forms import LoginForm, RegisterForm, StartWithIdForm from app.forms import LoginForm, RegisterForm, StartWithIdForm
from app.utils import saved_data_as_file, map_answers_to_questions
# Stimuli upload folder setting # Stimuli upload folder setting
APP_ROOT = os.path.dirname(os.path.abspath(__file__)) APP_ROOT = os.path.dirname(os.path.abspath(__file__))
...@@ -336,164 +334,7 @@ def view_research_notification(): ...@@ -336,164 +334,7 @@ def view_research_notification():
return render_template('view_research_notification.html', research_notification_filename=research_notification_filename) return render_template('view_research_notification.html', research_notification_filename=research_notification_filename)
@app.route('/download_csv')
@login_required
def download_csv():
exp_id = request.args.get('exp_id', None)
experiment_info = experiment.query.filter_by(idexperiment=exp_id).all()
print(experiment_info)
# answer sets with participant ids
participants = answer_set.query.filter_by(
experiment_idexperiment=exp_id).all()
# pages aka stimulants
pages = page.query.filter_by(experiment_idexperiment=exp_id).all()
# background questions
bg_questions = background_question.query.filter_by(
experiment_idexperiment=exp_id).all()
# question
questions = question.query.filter_by(experiment_idexperiment=exp_id).all()
# embody questions
embody_questions = embody_question.query.filter_by(
experiment_idexperiment=exp_id).all()
csv = ''
# create CSV-header
header = 'participant id;'
header += ';'.join([str(count) + '. bg_question: ' + question.background_question.strip()
for count, question in enumerate(bg_questions, 1)])
for idx in range(1, len(pages) + 1):
if len(questions) > 0:
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. slider_question: ' +
question.question.strip() for count, question in enumerate(questions, 1)])
for idx in range(1, len(pages) + 1):
if len(embody_questions) > 0:
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' +
question.picture.strip() for count, question in enumerate(embody_questions, 1)])
csv += header + '\r\n'
answer_row = ''
for participant in participants:
# list only finished answer sets
if participant.answer_counter > 0:
try:
# append user session id
answer_row += participant.session + ';'
# append background question answers
bg_answers = background_question_answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set).all()
bg_answers_list = [str(a.answer).strip() for a in bg_answers]
answer_row += ';'.join(bg_answers_list) + ';'
# append slider answers
slider_answers = answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set) \
.order_by(answer.page_idpage) \
.all()
pages_and_questions = {}
for p in pages:
questions_list = [(p.idpage, a.idquestion) for a in questions]
pages_and_questions[p.idpage] = questions_list
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
answers_list = map_answers_to_questions(slider_answers, _questions)
# typecast elemnts to string
answers_list = [str(a).strip() for a in answers_list]
answer_row += ';'.join(answers_list) + \
';' if slider_answers else len(
questions) * len(pages) * ';'
# append embody answers (coordinates)
# save embody answers as bitmap images
embody_answers = embody_answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set) \
.order_by(embody_answer.page_idpage) \
.all()
pages_and_questions = {}
for p in pages:
questions_list = [(p.idpage, a.idembody) for a in embody_questions]
pages_and_questions[p.idpage] = questions_list
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
_embody_answers = map_answers_to_questions(embody_answers, _questions)
answers_list = []
for answer_data in _embody_answers:
if not answer_data:
answers_list.append('')
continue
try:
coordinates = json.loads(answer_data.coordinates)
em_height = coordinates.get('height', 600) + 2
em_width = coordinates.get('width', 200) + 2
coordinates_to_bitmap = [
[0 for x in range(em_height)] for y in range(em_width)]
coordinates = list(
zip(coordinates.get('x'), coordinates.get('y')))
for point in coordinates:
try:
# for every brush stroke, increment the pixel
# value for every brush stroke
coordinates_to_bitmap[point[0]][point[1]] += 0.1
except IndexError:
continue
answers_list.append(json.dumps(coordinates_to_bitmap))
except ValueError as err:
app.logger(err)
answer_row += ';'.join(answers_list) if embody_answers else \
len(embody_questions) * len(pages) * ';'
# old way to save only visited points:
# answers_list = [json.dumps(
# list(zip( json.loads(a.coordinates)['x'],
# json.loads(a.coordinates)['y']))) for a in embody_answers]
except TypeError as err:
print(err)
csv += answer_row + '\r\n'
answer_row = ''
filename = "experiment_{}_{}.csv".format(
exp_id, date.today().strftime("%Y-%m-%d"))
return saved_data_as_file(filename, csv)
@app.route('/researcher_info') @app.route('/researcher_info')
@login_required @login_required
def researcher_info(): def researcher_info():
return render_template('researcher_info.html') return render_template('researcher_info.html')
# EOF
...@@ -75,4 +75,15 @@ body { ...@@ -75,4 +75,15 @@ body {
max-width: 90%; max-width: 90%;
} }
} }
\ No newline at end of file
#export-link-container {
margin-top: 20px;
padding: 10px;
}
#export-error {
float:right;
color:red;
}
$(document).ready(function()  {
var exportButton = $(".get-csv-results");