From 7bebb074081691f7d211e9cf63b81e03ceafc8d4 Mon Sep 17 00:00:00 2001
From: osmala <ossi.laine@utu.fi>
Date: Tue, 26 Jan 2021 11:36:54 +0200
Subject: [PATCH] Optimized file generation while exporting data to csv
---
app/experiment/views.py | 17 +++++++++--------
app/utils.py | 17 ++++++++---------
2 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/app/experiment/views.py b/app/experiment/views.py
index 711f3fa..4506e74 100644
--- a/app/experiment/views.py
+++ b/app/experiment/views.py
@@ -1062,24 +1062,25 @@ def start_download_csv():
def download_csv(meta):
exp_id = meta["exp_id"]
- data = generate_csv(exp_id)
# error handling
+ '''
if isinstance(data, Exception):
emit('timeout', {'exc': str(data)})
return
+ '''
# create temporary file
fd, path = mkstemp()
with os.fdopen(fd, 'w') as tmp:
- tmp.write(data)
- tmp.flush()
- # return path and filename to front so user can start downloading
- filename = "experiment_{}_{}".format(
- exp_id, date.today().strftime("%Y-%m-%d"))
- path = path.split('/')[-1]
- emit('file_ready', {'path': path, 'filename': filename})
+ if generate_csv(exp_id, tmp):
+
+ # return path and filename to front so user can start downloading
+ filename = "experiment_{}_{}".format(
+ exp_id, date.today().strftime("%Y-%m-%d"))
+ path = path.split('/')[-1]
+ emit('file_ready', {'path': path, 'filename': filename})
@socketio.on('end', namespace="/download_csv")
diff --git a/app/utils.py b/app/utils.py
index a64c872..36c1e32 100644
--- a/app/utils.py
+++ b/app/utils.py
@@ -109,7 +109,7 @@ def map_answers_to_questions(answers, questions):
@timeit
-def generate_csv(exp_id):
+def generate_csv(exp_id, file_handle):
# answer sets with participant ids
participants = answer_set.query.filter_by(
@@ -129,8 +129,6 @@ def generate_csv(exp_id):
embody_questions = embody_question.query.filter_by(
experiment_idexperiment=exp_id).all()
- csv = ''
-
# create CSV-header
header = 'participant id;'
header += ';'.join([str(count) + '. bg_question: ' + q.background_question.strip()
@@ -146,7 +144,7 @@ def generate_csv(exp_id):
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' +
question.picture.strip() for count, question in enumerate(embody_questions, 1)])
- csv += header + '\r\n'
+ file_handle.write(header + '\r\n')
# filter empty answer_sets
participants = list(filter(lambda participant: True if int(
@@ -167,16 +165,17 @@ def generate_csv(exp_id):
try:
emit('progress', {'done': nth, 'from': len_participants})
data = future.result()
- csv += data + '\r\n'
+ file_handle.write(data + '\n')
except Exception as exc:
print('generated an exception: {}'.format(exc))
- return exc
-
- return csv
+ emit('timeout', {'exc': str(data)})
+ return False
+
+ # file_handle.flush()
+ return True
def generate_answer_row(participant, pages, questions, embody_questions):
- # TODO: refactor
with app.app_context():
--
GitLab