Commit c3aab919 authored by Ossi Laine's avatar Ossi Laine
Browse files

Flush writes between every row when generating big CSV export file

parent 7bebb074
...@@ -1062,25 +1062,17 @@ def start_download_csv(): ...@@ -1062,25 +1062,17 @@ def start_download_csv():
def download_csv(meta): def download_csv(meta):
exp_id = meta["exp_id"] exp_id = meta["exp_id"]
# error handling
if isinstance(data, Exception):
emit('timeout', {'exc': str(data)})
# create temporary file # create temporary file
fd, path = mkstemp() fd, path = mkstemp()
with os.fdopen(fd, 'w') as tmp: with os.fdopen(fd, 'w', buffering=1) as tmp:
if generate_csv(exp_id, tmp): if generate_csv(exp_id, tmp):
# return path and filename to front so user can start downloading # return path and filename to front so user can start downloading
filename = "experiment_{}_{}".format( filename = "experiment_{}_{}".format(
exp_id,"%Y-%m-%d")) exp_id,"%Y-%m-%d"))
path = path.split('/')[-1] path = path.split('/')[-1]
emit('file_ready', {'path': path, 'filename': filename}) emit('file_ready', {'path': path, 'filename': filename})
emit('timeout', {'exc': 'job failed'})
@socketio.on('end', namespace="/download_csv") @socketio.on('end', namespace="/download_csv")
...@@ -166,12 +166,12 @@ def generate_csv(exp_id, file_handle): ...@@ -166,12 +166,12 @@ def generate_csv(exp_id, file_handle):
emit('progress', {'done': nth, 'from': len_participants}) emit('progress', {'done': nth, 'from': len_participants})
data = future.result() data = future.result()
file_handle.write(data + '\n') file_handle.write(data + '\n')
# to ensure that all internal buffers associated with fd are written to disk
except Exception as exc: except Exception as exc:
print('generated an exception: {}'.format(exc)) print('generated an exception: {}'.format(exc))
emit('timeout', {'exc': str(data)}) # return False
return False
# file_handle.flush()
return True return True
...@@ -263,7 +263,8 @@ def generate_answer_row(participant, pages, questions, embody_questions): ...@@ -263,7 +263,8 @@ def generate_answer_row(participant, pages, questions, embody_questions):
answers_list.append(json.dumps(coordinates_to_bitmap)) answers_list.append(json.dumps(coordinates_to_bitmap))
except ValueError as err: except ValueError as err:
app.logger(err) print(err)
answer_row += ';'.join(answers_list) if embody_answers else \ answer_row += ';'.join(answers_list) if embody_answers else \
len(embody_questions) * len(pages) * ';' len(embody_questions) * len(pages) * ';'
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment