There will be a short maintenance break on Wed 27.10. at 12:00. Estimated time 30 minutes.

Commit 108ee594 authored by Ossi Laine's avatar Ossi Laine
Browse files

Merge branch 'feat-optimize-downloading-dataset' into 'dev'

Feat optimize downloading dataset

See merge request tithei/pet-rating!4
parents 2c7fa1c8 912429a3
......@@ -15,7 +15,7 @@ from flask_cors import CORS, cross_origin
app = Flask(__name__)
CORS(app)
#CORS(app, resources={r"/*": {"cors_allowed_origins":"*"} } )
# CORS(app, resources={r"/*": {"cors_allowed_origins":"*"} } )
#app.config['BABEL_DEFAULT_LOCALE'] = 'fin'
#app.config['BABEL_TRANSLATION_DIRECTORIES'] ='C:/Users/Timo/git/pet-rating/app/translations'
......@@ -68,9 +68,6 @@ def get_locale():
"""
# Run flask app with socketIO
socketio = SocketIO()
socketio.init_app(app)
#mariabd mysql portti 3306 tarkista?
......@@ -81,6 +78,11 @@ migrate = Migrate(app, db)
login = LoginManager(app)
login.login_view = 'login'
# Run flask app with socketIO
socketio = SocketIO(app, cors_allowed_origins="*")
# socketio = SocketIO()
socketio.init_app(app)
# Register blueprints
from .task.views import task_blueprint
from .experiment.views import experiment_blueprint
......
......@@ -36,7 +36,25 @@
<tr>
<td>Number of finished ratings:</td>
<td>{{ finished_ratings }}
<a class="btn btn-primary btn-info float-right" href="{{ url_for('download_csv', exp_id=exp.idexperiment) }}" role="button">Export results (csv)</a>
<button data-value="{{ exp.idexperiment }}" class="btn btn-primary float-right get-csv-results">
export results
</button>
<div id="export-link-container" class="hidden">
<a id="export-link" class="float-right" href="{{ url_for('experiment.download_csv', exp_id=exp.idexperiment) }}" role="button"></a>
<p id="export-error"></p>
</div>
<div class="progress hidden">
<div id="export-results-bar" class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" aria-valuenow="50" aria-valuemin="0" aria-valuemax="100" style="width: 0%">
</div>
</div>
</td>
</tr>
</tbody>
......@@ -194,6 +212,8 @@
</table>
<script src="{{ url_for('static', filename='lib/js/socket.io.js') }}" ></script>
<script src="{{ url_for('static', filename='js/urls.js') }}" ></script>
<script src="{{ url_for('static', filename='js/getDrawing.js') }}" ></script>
<script src="{{ url_for('static', filename='js/getCSV.js') }}" ></script>
{% endblock %}
from flask_cors import CORS, cross_origin
from app import socketio
from flask_socketio import emit
import embody_plot
import os
import secrets
import json
from datetime import date
from tempfile import mkstemp
from flask_socketio import emit
from sqlalchemy import and_
from flask_login import login_required
from werkzeug import secure_filename
from flask import (
Flask,
render_template,
request,
session,
flash,
redirect,
url_for,
Blueprint,
jsonify
send_file
)
from wtforms import Form
from sqlalchemy import and_, update
from flask_login import login_required
from werkzeug import secure_filename
from app import app, db
from app import app, db, socketio
from app.routes import APP_ROOT
from app.models import background_question, experiment
from app.models import background_question_answer
from app.models import page, question
from app.models import background_question_option
from app.models import answer_set, answer, forced_id
from app.models import user, trial_randomization
from app.models import trial_randomization
from app.models import embody_answer, embody_question
from app.forms import (
CreateBackgroundQuestionForm,
......@@ -40,8 +33,10 @@ from app.forms import (
EditQuestionForm, EditExperimentForm, UploadResearchBulletinForm,
EditPageForm, RemoveExperimentForm, GenerateIdForm, CreateEmbodyForm
)
from app.utils import get_mean_from_slider_answers, map_answers_to_questions
from app.utils import get_mean_from_slider_answers, map_answers_to_questions, \
generate_csv
import embody_plot
# Stimuli upload folder setting
#APP_ROOT = os.path.dirname(os.path.abspath(__file__))
......@@ -957,6 +952,8 @@ def statistics():
questions = question.query.filter_by(experiment_idexperiment=exp_id).all()
pages_and_questions = {}
'''
for p in pages:
questions_list = [(p.idpage, a.idquestion) for a in questions]
pages_and_questions[p.idpage] = questions_list
......@@ -965,27 +962,34 @@ def statistics():
# those are in answer table as page_idpage and question_idquestion respectively
slider_answers = {}
for participant in participants:
if participant.answer_counter > 0:
answers = answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set)\
.order_by(answer.page_idpage)\
.all()
# flatten pages and questions to list of tuples (page_id, question_id)
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
if int(participant.answer_counter) == 0:
continue
answers = answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set)\
.order_by(answer.page_idpage)\
.all()
slider_answers[participant.session] = map_answers_to_questions(
answers, _questions)
# flatten pages and questions to list of tuples (page_id, question_id)
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
slider_answers[participant.session] = map_answers_to_questions(
answers, _questions)
mean = get_mean_from_slider_answers(slider_answers)
# slider_answers['mean'] = get_mean_from_slider_answers(slider_answers)
slider_answers = {
'mean': mean
'mean': mean
}
'''
slider_answers = {}
# Background question answers
bg_questions = background_question.query.filter_by(
experiment_idexperiment=exp_id).all()
......@@ -1012,8 +1016,27 @@ def statistics():
finished_ratings=finished_ratings,
question_headers=question_headers,
stimulus_headers=stimulus_headers,
embody_questions=embody_questions
)
embody_questions=embody_questions)
@experiment_blueprint.route('/download_csv')
def download_csv():
exp_id = request.args.get('exp_id', None)
path = request.args.get('path', None)
filename = "experiment_{}_{}.csv".format(
exp_id, date.today().strftime("%Y-%m-%d"))
path = '/tmp/' + path
try:
return send_file(path,
mimetype='text/csv',
as_attachment=True,
attachment_filename=filename)
finally:
os.remove(path)
def remove_rows(rows):
......@@ -1024,27 +1047,55 @@ def remove_rows(rows):
@socketio.on('connect', namespace="/create_embody")
def create_embody():
def start_create_embody():
emit('success', {'connection': 'on'})
@socketio.on('draw', namespace="/create_embody")
def create_embody(page_id):
print("DRAW")
page = page_id["page"]
embody = page_id["embody"]
def create_embody(meta):
page = meta["page"]
embody = meta["embody"]
img_path = embody_plot.get_coordinates(page, embody)
app.logger.info(img_path)
emit('end', {'path': img_path})
@socketio.on('end', namespace="/create_embody")
def create_embody():
print("connection end")
emit('end', {'connection': 'off'})
@socketio.on('connect', namespace="/download_csv")
def start_download_csv():
emit('success', {'connection': 'Start generating CSV file'})
# EOF
@socketio.on('generate_csv', namespace="/download_csv")
def download_csv(meta):
exp_id = meta["exp_id"]
data = generate_csv(exp_id)
# error handling
if isinstance(data, Exception):
emit('timeout', {'exc': str(data)})
return
# create temporary file
fd, path = mkstemp()
with os.fdopen(fd, 'w') as tmp:
tmp.write(data)
tmp.flush()
# return path and filename to front so user can start downloading
filename = "experiment_{}_{}".format(
exp_id, date.today().strftime("%Y-%m-%d"))
path = path.split('/')[-1]
emit('file_ready', {'path': path, 'filename': filename})
@socketio.on('end', namespace="/download_csv")
def end_download_csv():
# TODO: not working solution... db session keeps hanging after socket session has ended
# mysqld timeout is set to 180s, so it kills hanging connections, but this is not a good solution
db.session.close()
@socketio.on('end', namespace="/create_embody")
def end_create_embody():
db.session.close()
......@@ -153,6 +153,12 @@ class answer (db.Model):
answer = db.Column(db.String(120))
page_idpage = db.Column(db.Integer, db.ForeignKey('page.idpage'))
def question(self):
return int(self.question_idquestion)
def result(self):
return int(self.answer)
def __repr__(self):
return "<idanswer = '%s', question_idquestion = '%s', answer_set_idanswer_set = '%s', answer = '%s', page_idpage = '%s'>" % (self.idanswer, self.question_idquestion, self.answer_set_idanswer_set, self.answer, self.page_idpage)
......@@ -167,6 +173,12 @@ class embody_answer (db.Model):
db.Integer, db.ForeignKey('embody_question.idembody'))
coordinates = db.Column(db.Text)
def question(self):
return self.embody_question_idembody
def result(self):
return self.coordinates
def __repr__(self):
return "<idanswer = '%s', answer_set_idanswer_set = '%s', coordinates = '%s', page_idpage = '%s', embody_question_idembody='%s' >" % (self.idanswer, self.answer_set_idanswer_set, self.coordinates, self.page_idpage, self.embody_question_idembody)
......
import os
import random
import secrets
from datetime import datetime, date
import json
from datetime import datetime
from flask import (render_template,
request,
......@@ -16,12 +15,11 @@ from flask_login import current_user, login_user, logout_user, login_required
from app import app, db
from app.models import background_question, experiment
from app.models import background_question_answer
from app.models import page, question, embody_question, embody_answer
from app.models import page
from app.models import background_question_option
from app.models import answer_set, answer, forced_id
from app.models import answer_set, forced_id
from app.models import user, trial_randomization
from app.forms import LoginForm, RegisterForm, StartWithIdForm
from app.utils import saved_data_as_file, map_answers_to_questions
# Stimuli upload folder setting
APP_ROOT = os.path.dirname(os.path.abspath(__file__))
......@@ -336,164 +334,7 @@ def view_research_notification():
return render_template('view_research_notification.html', research_notification_filename=research_notification_filename)
@app.route('/download_csv')
@login_required
def download_csv():
exp_id = request.args.get('exp_id', None)
experiment_info = experiment.query.filter_by(idexperiment=exp_id).all()
print(experiment_info)
# answer sets with participant ids
participants = answer_set.query.filter_by(
experiment_idexperiment=exp_id).all()
# pages aka stimulants
pages = page.query.filter_by(experiment_idexperiment=exp_id).all()
# background questions
bg_questions = background_question.query.filter_by(
experiment_idexperiment=exp_id).all()
# question
questions = question.query.filter_by(experiment_idexperiment=exp_id).all()
# embody questions
embody_questions = embody_question.query.filter_by(
experiment_idexperiment=exp_id).all()
csv = ''
# create CSV-header
header = 'participant id;'
header += ';'.join([str(count) + '. bg_question: ' + question.background_question.strip()
for count, question in enumerate(bg_questions, 1)])
for idx in range(1, len(pages) + 1):
if len(questions) > 0:
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. slider_question: ' +
question.question.strip() for count, question in enumerate(questions, 1)])
for idx in range(1, len(pages) + 1):
if len(embody_questions) > 0:
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' +
question.picture.strip() for count, question in enumerate(embody_questions, 1)])
csv += header + '\r\n'
answer_row = ''
for participant in participants:
# list only finished answer sets
if participant.answer_counter > 0:
try:
# append user session id
answer_row += participant.session + ';'
# append background question answers
bg_answers = background_question_answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set).all()
bg_answers_list = [str(a.answer).strip() for a in bg_answers]
answer_row += ';'.join(bg_answers_list) + ';'
# append slider answers
slider_answers = answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set) \
.order_by(answer.page_idpage) \
.all()
pages_and_questions = {}
for p in pages:
questions_list = [(p.idpage, a.idquestion) for a in questions]
pages_and_questions[p.idpage] = questions_list
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
answers_list = map_answers_to_questions(slider_answers, _questions)
# typecast elemnts to string
answers_list = [str(a).strip() for a in answers_list]
answer_row += ';'.join(answers_list) + \
';' if slider_answers else len(
questions) * len(pages) * ';'
# append embody answers (coordinates)
# save embody answers as bitmap images
embody_answers = embody_answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set) \
.order_by(embody_answer.page_idpage) \
.all()
pages_and_questions = {}
for p in pages:
questions_list = [(p.idpage, a.idembody) for a in embody_questions]
pages_and_questions[p.idpage] = questions_list
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
_embody_answers = map_answers_to_questions(embody_answers, _questions)
answers_list = []
for answer_data in _embody_answers:
if not answer_data:
answers_list.append('')
continue
try:
coordinates = json.loads(answer_data.coordinates)
em_height = coordinates.get('height', 600) + 2
em_width = coordinates.get('width', 200) + 2
coordinates_to_bitmap = [
[0 for x in range(em_height)] for y in range(em_width)]
coordinates = list(
zip(coordinates.get('x'), coordinates.get('y')))
for point in coordinates:
try:
# for every brush stroke, increment the pixel
# value for every brush stroke
coordinates_to_bitmap[point[0]][point[1]] += 0.1
except IndexError:
continue
answers_list.append(json.dumps(coordinates_to_bitmap))
except ValueError as err:
app.logger(err)
answer_row += ';'.join(answers_list) if embody_answers else \
len(embody_questions) * len(pages) * ';'
# old way to save only visited points:
# answers_list = [json.dumps(
# list(zip( json.loads(a.coordinates)['x'],
# json.loads(a.coordinates)['y']))) for a in embody_answers]
except TypeError as err:
print(err)
csv += answer_row + '\r\n'
answer_row = ''
filename = "experiment_{}_{}.csv".format(
exp_id, date.today().strftime("%Y-%m-%d"))
return saved_data_as_file(filename, csv)
@app.route('/researcher_info')
@login_required
def researcher_info():
return render_template('researcher_info.html')
# EOF
......@@ -75,4 +75,15 @@ body {
max-width: 90%;
}
}
\ No newline at end of file
}
#export-link-container {
margin-top: 20px;
padding: 10px;
}
#export-error {
float:right;
color:red;
}
$(document).ready(function()  {
var exportButton = $(".get-csv-results");
var progressBarContainer = $(".progress")
var progressBar = $("#export-results-bar")
var exportLinkContainer = $("#export-link-container");
var exportLink = $("#export-link");
var exportError = $("#export-error");
// With sockets
function initConnection(socket) {
socket.on('success', function(msg) {
exportButton.text('Generating file...')
exportButton.addClass('disabled')
});
socket.on('progress', function(data) {
progressBar.width(100*(data.done/data.from) + '%')
});
socket.on('timeout', function(data) {
// kill connection
socket.emit('end')
socket.disconnect()
exportButton.text('Export results')
exportButton.removeClass('disabled')
progressBarContainer.addClass("hidden")
// show error
exportLinkContainer.removeClass("hidden")
exportError.text('Error: ' + data.exc)
});
socket.on('file_ready', function(file) {
socket.emit('end')
socket.disconnect()
exportButton.text('File is ready!')
// show link
exportLinkContainer.removeClass("hidden")
exportLink.text('Download: ' + file.filename + '.csv')
// set filename to exportlink
var href = exportLink.attr('href');
href += '&path=' + file.path
$(exportLink).attr('href', href);
// Remove progress bar
progressBarContainer.addClass("hidden")
progressBar.width('0%')
});
}
exportButton.click(function(event) {
event.preventDefault()
// Init socket
var socket = io.connect(exportURL);
initConnection(socket)
// start generating csv file...
socket.emit('generate_csv', {exp_id: this.dataset.value})
progressBarContainer.removeClass("hidden")
})
})
const baseURI = 'localhost/';
//const baseURI = 'http://onni.utu.fi/';
var getDrawingURI = baseURI + 'create_embody';
$(document).ready(function()  {
var drawButtons = $(".embody-get-drawing");
......@@ -23,7 +19,8 @@ $(document).ready(function()  {
});
socket.on('end', function(img) {
// kill connection
socket.emit('end')
socket.disconnect()
// Draw image to statistic -page
......@@ -45,37 +42,13 @@ $(document).ready(function()  {
var socket = io.connect(getDrawingURI);
initConnection(socket)
//
var pageId = this.dataset.value.split('-')[0]
var embodyId = this.dataset.value.split('-')[1]
console.log(pageId)
console.log(embodyId)
socket.emit('draw', {page:pageId, embody:embodyId})