From c3aab9193bb2a25daba9650b13c3314294c1e3e9 Mon Sep 17 00:00:00 2001
From: osmala <ossi.laine@utu.fi>
Date: Tue, 26 Jan 2021 17:28:16 +0200
Subject: [PATCH] Flush writes between every row when generating big CSV export
file
---
app/experiment/views.py | 14 +++-----------
app/utils.py | 9 +++++----
2 files changed, 8 insertions(+), 15 deletions(-)
diff --git a/app/experiment/views.py b/app/experiment/views.py
index 4506e74..3bf8aeb 100644
--- a/app/experiment/views.py
+++ b/app/experiment/views.py
@@ -1062,25 +1062,17 @@ def start_download_csv():
def download_csv(meta):
exp_id = meta["exp_id"]
-
- # error handling
- '''
- if isinstance(data, Exception):
- emit('timeout', {'exc': str(data)})
- return
- '''
-
# create temporary file
fd, path = mkstemp()
- with os.fdopen(fd, 'w') as tmp:
-
+ with os.fdopen(fd, 'w', buffering=1) as tmp:
if generate_csv(exp_id, tmp):
-
# return path and filename to front so user can start downloading
filename = "experiment_{}_{}".format(
exp_id, date.today().strftime("%Y-%m-%d"))
path = path.split('/')[-1]
emit('file_ready', {'path': path, 'filename': filename})
+ else:
+ emit('timeout', {'exc': 'job failed'})
@socketio.on('end', namespace="/download_csv")
diff --git a/app/utils.py b/app/utils.py
index 36c1e32..9fc8802 100644
--- a/app/utils.py
+++ b/app/utils.py
@@ -166,12 +166,12 @@ def generate_csv(exp_id, file_handle):
emit('progress', {'done': nth, 'from': len_participants})
data = future.result()
file_handle.write(data + '\n')
+ # to ensure that all internal buffers associated with fd are written to disk
+ file_handle.flush()
except Exception as exc:
print('generated an exception: {}'.format(exc))
- emit('timeout', {'exc': str(data)})
- return False
+ # return False
- # file_handle.flush()
return True
@@ -263,7 +263,8 @@ def generate_answer_row(participant, pages, questions, embody_questions):
answers_list.append(json.dumps(coordinates_to_bitmap))
except ValueError as err:
- app.logger(err)
+ print(err)
+ #app.logger(err)
answer_row += ';'.join(answers_list) if embody_answers else \
len(embody_questions) * len(pages) * ';'
--
GitLab