Skip to content
Snippets Groups Projects
Commit 47c09e6f authored by Ossi Laine's avatar Ossi Laine
Browse files

Handle CSV export with socket connection in a background job

parent de0fd379
Branches
Tags
No related merge requests found
......@@ -15,7 +15,7 @@ from flask_cors import CORS, cross_origin
app = Flask(__name__)
CORS(app)
#CORS(app, resources={r"/*": {"cors_allowed_origins":"*"} } )
# CORS(app, resources={r"/*": {"cors_allowed_origins":"*"} } )
#app.config['BABEL_DEFAULT_LOCALE'] = 'fin'
#app.config['BABEL_TRANSLATION_DIRECTORIES'] ='C:/Users/Timo/git/pet-rating/app/translations'
......@@ -69,7 +69,8 @@ def get_locale():
# Run flask app with socketIO
socketio = SocketIO()
socketio = SocketIO(app, cors_allowed_origins="*")
# socketio = SocketIO()
socketio.init_app(app)
#mariabd mysql portti 3306 tarkista?
......
......@@ -36,7 +36,24 @@
<tr>
<td>Number of finished ratings:</td>
<td>{{ finished_ratings }}
<a class="btn btn-primary btn-info float-right" href="{{ url_for('download_csv', exp_id=exp.idexperiment) }}" role="button">Export results (csv)</a>
<button data-value="{{ exp.idexperiment }}" class="btn btn-primary float-right get-csv-results">
export results
</button>
<div id="export-link-container" class="hidden">
<a id="export-link" class="float-right" href="{{ url_for('experiment.download_csv', exp_id=exp.idexperiment) }}" role="button">Download</a>
</div>
<div class="progress hidden">
<div id="export-results-bar" class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" aria-valuenow="50" aria-valuemin="0" aria-valuemax="100" style="width: 0%">
</div>
</div>
</td>
</tr>
</tbody>
......@@ -194,6 +211,8 @@
</table>
<script src="{{ url_for('static', filename='lib/js/socket.io.js') }}" ></script>
<script src="{{ url_for('static', filename='js/urls.js') }}" ></script>
<script src="{{ url_for('static', filename='js/getDrawing.js') }}" ></script>
<script src="{{ url_for('static', filename='js/getCSV.js') }}" ></script>
{% endblock %}
from flask_cors import CORS, cross_origin
from app import socketio
from flask_socketio import emit
import embody_plot
import os
import secrets
import json
from datetime import datetime, date
from flask import (
Flask,
......@@ -40,8 +40,8 @@ from app.forms import (
EditQuestionForm, EditExperimentForm, UploadResearchBulletinForm,
EditPageForm, RemoveExperimentForm, GenerateIdForm, CreateEmbodyForm
)
from app.utils import get_mean_from_slider_answers, map_answers_to_questions
from app.utils import get_mean_from_slider_answers, map_answers_to_questions, \
saved_data_as_file, timeit, generate_csv
# Stimuli upload folder setting
#APP_ROOT = os.path.dirname(os.path.abspath(__file__))
......@@ -1032,13 +1032,9 @@ def create_embody():
@socketio.on('draw', namespace="/create_embody")
def create_embody(page_id):
print("DRAW")
page = page_id["page"]
embody = page_id["embody"]
def create_embody(meta):
page = meta["page"]
embody = meta["embody"]
img_path = embody_plot.get_coordinates(page, embody)
app.logger.info(img_path)
emit('end', {'path': img_path})
......@@ -1046,8 +1042,64 @@ def create_embody(page_id):
@socketio.on('end', namespace="/create_embody")
def create_embody():
print("connection end")
emit('end', {'connection': 'off'})
# EOF
@socketio.on('connect', namespace="/download_csv")
def create_embody():
emit('success', {'connection': 'Start generating CSV file'})
from tempfile import mkstemp
from flask import send_file
@socketio.on('generate_csv', namespace="/download_csv")
def create_embody(meta):
exp_id = meta["exp_id"]
csv = generate_csv(exp_id)
if not csv:
emit('timeout')
filename = "experiment_{}_{}".format(
exp_id, date.today().strftime("%Y-%m-%d"))
fd, path = mkstemp()
print(fd)
print(path)
with os.fdopen(fd, 'w') as tmp:
tmp.write(csv)
tmp.flush()
path = path.split('/')[-1]
emit('file_ready', {'path': path, 'filename': filename})
# return saved_data_as_file(filename, csv)
@experiment_blueprint.route('/download_csv')
def download_csv():
exp_id = request.args.get('exp_id', None)
path = request.args.get('path', None)
filename = "experiment_{}_{}.csv".format(
exp_id, date.today().strftime("%Y-%m-%d"))
path = '/tmp/' + path
try:
return send_file(path,
mimetype='text/csv',
as_attachment=True,
attachment_filename=filename)
finally:
os.remove(path)
......@@ -153,6 +153,12 @@ class answer (db.Model):
answer = db.Column(db.String(120))
page_idpage = db.Column(db.Integer, db.ForeignKey('page.idpage'))
def question(self):
return int(self.question_idquestion)
def result(self):
return int(self.answer)
def __repr__(self):
return "<idanswer = '%s', question_idquestion = '%s', answer_set_idanswer_set = '%s', answer = '%s', page_idpage = '%s'>" % (self.idanswer, self.question_idquestion, self.answer_set_idanswer_set, self.answer, self.page_idpage)
......@@ -167,6 +173,12 @@ class embody_answer (db.Model):
db.Integer, db.ForeignKey('embody_question.idembody'))
coordinates = db.Column(db.Text)
def question(self):
return self.embody_question_idembody
def result(self):
return self.coordinates
def __repr__(self):
return "<idanswer = '%s', answer_set_idanswer_set = '%s', coordinates = '%s', page_idpage = '%s', embody_question_idembody='%s' >" % (self.idanswer, self.answer_set_idanswer_set, self.coordinates, self.page_idpage, self.embody_question_idembody)
......
......@@ -21,7 +21,6 @@ from app.models import background_question_option
from app.models import answer_set, answer, forced_id
from app.models import user, trial_randomization
from app.forms import LoginForm, RegisterForm, StartWithIdForm
from app.utils import saved_data_as_file, map_answers_to_questions, timeit
# Stimuli upload folder setting
APP_ROOT = os.path.dirname(os.path.abspath(__file__))
......@@ -336,181 +335,7 @@ def view_research_notification():
return render_template('view_research_notification.html', research_notification_filename=research_notification_filename)
@timeit
@app.route('/download_csv')
@login_required
def download_csv():
exp_id = request.args.get('exp_id', None)
experiment_info = experiment.query.filter_by(idexperiment=exp_id).all()
print(experiment_info)
# answer sets with participant ids
participants = answer_set.query.filter_by(
experiment_idexperiment=exp_id).all()
# pages aka stimulants
pages = page.query.filter_by(experiment_idexperiment=exp_id).all()
# background questions
bg_questions = background_question.query.filter_by(
experiment_idexperiment=exp_id).all()
# question
questions = question.query.filter_by(experiment_idexperiment=exp_id).all()
# embody questions
embody_questions = embody_question.query.filter_by(
experiment_idexperiment=exp_id).all()
csv = ''
# create CSV-header
header = 'participant id;'
header += ';'.join([str(count) + '. bg_question: ' + q.background_question.strip()
for count, q in enumerate(bg_questions, 1)])
for idx in range(1, len(pages) + 1):
if len(questions) > 0:
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. slider_question: ' +
question.question.strip() for count, question in enumerate(questions, 1)])
for idx in range(1, len(pages) + 1):
if len(embody_questions) > 0:
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' +
question.picture.strip() for count, question in enumerate(embody_questions, 1)])
csv += header + '\r\n'
csv += generate_answers(participants, pages, questions, embody_questions)
filename = "experiment_{}_{}.csv".format(
exp_id, date.today().strftime("%Y-%m-%d"))
return saved_data_as_file(filename, csv)
@timeit
def generate_answers(participants, pages, questions, embody_questions):
csv = ''
answer_row = ''
for participant in participants:
# list only finished answer sets
if int(participant.answer_counter) == 0:
continue
try:
# append user session id
answer_row += participant.session + ';'
# append background question answers
bg_answers = background_question_answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set).all()
bg_answers_list = [str(a.answer).strip() for a in bg_answers]
answer_row += ';'.join(bg_answers_list) + ';'
# append slider answers
slider_answers = answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set) \
.order_by(answer.page_idpage, answer.question_idquestion) \
.all()
pages_and_questions = {}
for p in pages:
questions_list = [(p.idpage, a.idquestion) for a in questions]
pages_and_questions[p.idpage] = questions_list
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
answers_list = map_answers_to_questions(slider_answers, _questions)
# typecast elemnts to string
answers_list = [str(a).strip() for a in answers_list]
answer_row += ';'.join(answers_list) + \
';' if slider_answers else len(
questions) * len(pages) * ';'
# append embody answers (coordinates)
# save embody answers as bitmap images
embody_answers = embody_answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set) \
.order_by(embody_answer.page_idpage) \
.all()
pages_and_questions = {}
for p in pages:
questions_list = [(p.idpage, a.idembody) for a in embody_questions]
pages_and_questions[p.idpage] = questions_list
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
_embody_answers = map_answers_to_questions(embody_answers, _questions)
answers_list = []
for answer_data in _embody_answers:
if not answer_data:
answers_list.append('')
continue
try:
coordinates = json.loads(answer_data.coordinates)
em_height = coordinates.get('height', 600) + 2
em_width = coordinates.get('width', 200) + 2
coordinates_to_bitmap = [
[0 for x in range(em_height)] for y in range(em_width)]
coordinates = list(
zip(coordinates.get('x'), coordinates.get('y')))
for point in coordinates:
try:
# for every brush stroke, increment the pixel
# value for every brush stroke
coordinates_to_bitmap[point[0]][point[1]] += 0.1
except IndexError:
continue
answers_list.append(json.dumps(coordinates_to_bitmap))
except ValueError as err:
app.logger(err)
answer_row += ';'.join(answers_list) if embody_answers else \
len(embody_questions) * len(pages) * ';'
# old way to save only visited points:
# answers_list = [json.dumps(
# list(zip( json.loads(a.coordinates)['x'],
# json.loads(a.coordinates)['y']))) for a in embody_answers]
except TypeError as err:
print(err)
csv += answer_row + '\r\n'
answer_row = ''
return csv
@app.route('/researcher_info')
@login_required
def researcher_info():
return render_template('researcher_info.html')
# EOF
......@@ -75,4 +75,10 @@ body {
max-width: 90%;
}
}
\ No newline at end of file
}
#export-link-container {
margin-top: 20px;
padding: 10px;
}
const baseURI = 'localhost/';
//const baseURI = 'http://onni.utu.fi/';
var getDrawingURI = baseURI + 'create_embody';
$(document).ready(function() {
var drawButtons = $(".embody-get-drawing");
......@@ -23,7 +19,6 @@ $(document).ready(function()  {
});
socket.on('end', function(img) {
socket.disconnect()
// Draw image to statistic -page
......
......@@ -2,10 +2,11 @@ import os
import tempfile
import time
from itertools import zip_longest
import concurrent.futures
from flask import send_file
from app import app
from app.models import background_question, background_question_answer, \
page, question, answer_set, answer, embody_answer, embody_question
def timeit(method):
......@@ -17,7 +18,7 @@ def timeit(method):
name = kw.get('log_name', method.__name__.upper())
kw['log_time'][name] = int((te - ts) * 1000)
else:
print('{} {:2.2f} ms'.format(method.__name__, (te - ts) * 1000))
app.logger.info('{} {:2.2f} ms'.format(method.__name__, (te - ts) * 1000))
return result
return timed
......@@ -71,7 +72,7 @@ def get_values_from_list_of_answers(page_question, answers):
def question_matches_answer(question, answer):
if (answer.page_idpage == question[0] and answer.question_idquestion == question[1]):
if (answer.page_idpage == question[0] and answer.question() == question[1]):
return True
return False
......@@ -98,7 +99,7 @@ def map_answers_to_questions(answers, questions):
break
if question_matches_answer(question, current_answer):
results[nth_question] = int(current_answer.answer)
results[nth_question] = current_answer.result
nth_answer += 1
return results
......@@ -136,4 +137,163 @@ group by sub.answer_set_idanswer_set;
# all possible page/question comobs
select distinct p.idpage, q.idquestion from question q join page p on p.experiment_idexperiment=q.experiment_idexperiment where p.experiment_idexperiment = 2 order by p.idpage,q.idquestion;
'''
\ No newline at end of file
'''
@timeit
def generate_csv(exp_id):
# answer sets with participant ids
participants = answer_set.query.filter_by(
experiment_idexperiment=exp_id).all()
# pages aka stimulants
pages = page.query.filter_by(experiment_idexperiment=exp_id).all()
# background questions
bg_questions = background_question.query.filter_by(
experiment_idexperiment=exp_id).all()
# question
questions = question.query.filter_by(experiment_idexperiment=exp_id).all()
# embody questions
embody_questions = embody_question.query.filter_by(
experiment_idexperiment=exp_id).all()
csv = ''
# create CSV-header
header = 'participant id;'
header += ';'.join([str(count) + '. bg_question: ' + q.background_question.strip()
for count, q in enumerate(bg_questions, 1)])
for idx in range(1, len(pages) + 1):
if len(questions) > 0:
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. slider_question: ' +
question.question.strip() for count, question in enumerate(questions, 1)])
for idx in range(1, len(pages) + 1):
if len(embody_questions) > 0:
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' +
question.picture.strip() for count, question in enumerate(embody_questions, 1)])
csv += header + '\r\n'
# filter empty answer_sets
participants = list(filter(lambda participant: True if int(
participant.answer_counter) > 0 else False, participants))
# We can use a with statement to ensure threads are cleaned up promptly
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
# Start the load operations and mark each future with its URL
future_to_answer = {
executor.submit(generate_answer_row, participant, pages, questions, embody_questions): participant
for participant in participants}
for future in concurrent.futures.as_completed(future_to_answer):
# for testing purpose
# answer_row = future_to_answer[future]
try:
data = future.result()
csv += data + '\r\n'
except TimeoutError:
return None
except Exception as exc:
print('generated an exception: {}'.format(exc))
return csv
def generate_answer_row(participant, pages, questions, embody_questions):
# TODO: refactor
answer_row = ''
# append user session id
answer_row += participant.session + ';'
# append background question answers
bg_answers = background_question_answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set).all()
bg_answers_list = [str(a.answer).strip() for a in bg_answers]
answer_row += ';'.join(bg_answers_list) + ';'
# append slider answers
slider_answers = answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set) \
.order_by(answer.page_idpage, answer.question_idquestion) \
.all()
pages_and_questions = {}
for p in pages:
questions_list = [(p.idpage, a.idquestion) for a in questions]
pages_and_questions[p.idpage] = questions_list
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
answers_list = map_answers_to_questions(slider_answers, _questions)
# typecast elemnts to string
answers_list = [str(a).strip() for a in answers_list]
answer_row += ';'.join(answers_list) + \
';' if slider_answers else len(
questions) * len(pages) * ';'
# append embody answers (coordinates)
# save embody answers as bitmap images
embody_answers = embody_answer.query.filter_by(
answer_set_idanswer_set=participant.idanswer_set) \
.order_by(embody_answer.page_idpage) \
.all()
pages_and_questions = {}
for p in pages:
questions_list = [(p.idpage, a.idembody) for a in embody_questions]
pages_and_questions[p.idpage] = questions_list
_questions = [
item for sublist in pages_and_questions.values() for item in sublist]
_embody_answers = map_answers_to_questions(embody_answers, _questions)
answers_list = []
for answer_data in _embody_answers:
if not answer_data:
answers_list.append('')
continue
try:
coordinates = json.loads(answer_data)
em_height = coordinates.get('height', 600) + 2
em_width = coordinates.get('width', 200) + 2
coordinates_to_bitmap = [
[0 for x in range(em_height)] for y in range(em_width)]
coordinates = list(
zip(coordinates.get('x'), coordinates.get('y')))
for point in coordinates:
try:
# for every brush stroke, increment the pixel
# value for every brush stroke
coordinates_to_bitmap[point[0]][point[1]] += 0.1
except IndexError:
continue
answers_list.append(json.dumps(coordinates_to_bitmap))
except ValueError as err:
app.logger(err)
answer_row += ';'.join(answers_list) if embody_answers else \
len(embody_questions) * len(pages) * ';'
return answer_row
from decouple import config
import os
basedir = os.path.abspath(os.path.dirname(__file__))
from decouple import config
class Config(object):
#seret key is set in __ini__.py
#SECRET_KEY = os.environ.get('SECRET_KEY') or 'you-will-never-guess'
# seret key is set in __ini__.py
#SECRET_KEY = os.environ.get('SECRET_KEY') or 'you-will-never-guess'
LANGUAGES = ['en', 'fi', 'fa', 'el', 'it', 'zh']
......@@ -19,23 +18,25 @@ class Config(object):
SQLALCHEMY_TRACK_MODIFICATIONS = False
"""
#MariaDB mysql database settings
# MariaDB mysql database settings
MYSQL_USER = config('MYSQL_USER')
MYSQL_PASSWORD = config('MYSQL_PASSWORD')
MYSQL_SERVER = config('MYSQL_SERVER')
MYSQL_DB = config('MYSQL_DB')
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://'+MYSQL_USER+':'+MYSQL_PASSWORD+'@'+MYSQL_SERVER+'/'+MYSQL_DB+'?charset=utf8mb4'
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://'+MYSQL_USER+':' + \
MYSQL_PASSWORD+'@'+MYSQL_SERVER+'/'+MYSQL_DB+'?charset=utf8mb4'
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_ENGINE_OPTIONS = {
"pool_pre_ping": True,
"pool_recycle": 300,
"max_overflow": 30,
"pool_size": 20
"pool_pre_ping": True,
"pool_recycle": 300,
"max_overflow": 30,
"pool_size": 20
}
TEMPLATES_AUTO_RELOAD = True
DEBUG = False
DEBUG = True
......@@ -234,7 +234,7 @@ def plot_coordinates(coordinates, image_path=DEFAULT_IMAGE_PATH):
ax1.plot(coordinates["x"],coordinates["y"], 'ro', alpha=0.2)
ax1.imshow(image, alpha=0.6)
app.logger.info("iamge plotted")
app.logger.info("image plotted")
# return figure for saving/etc...
return fig
......
......@@ -13,7 +13,8 @@ Flask-Cors==3.0.7
Flask-Login==0.4.1
Flask-Migrate==2.2.1
Flask-Session==0.3.1
Flask-SocketIO==3.3.2
Flask-SocketIO==4.3.0
# Flask-SocketIO==3.3.2
Flask-SQLAlchemy==2.3.2
Flask-Uploads==0.2.1
Flask-WTF==0.14.2
......@@ -40,8 +41,10 @@ PyMySQL==0.9.3
pyparsing==2.3.1
python-dateutil==2.7.3
python-editor==1.0.3
python-engineio==3.5.1
python-socketio==3.1.2
python-engineio==3.13.0
# python-engineio==3.5.1
# python-socketio==3.1.2
python-socketio==4.6.0
pytz==2018.7
rope==0.12.0
scipy==1.2.1
......@@ -54,4 +57,4 @@ visitor==0.1.3
Werkzeug==0.14.1
WTForms==2.2.1
WTForms-SQLAlchemy==0.1
python-decouple
\ No newline at end of file
python-decouple
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment