Commit 9968f630 authored by Ossi Laine's avatar Ossi Laine
Browse files

Merge branch 'feat-optimize-data-export' into 'master'

Feat optimize data export

See merge request tithei/pet-rating!8
parents 1311884b c3aab919
...@@ -1062,24 +1062,17 @@ def start_download_csv(): ...@@ -1062,24 +1062,17 @@ def start_download_csv():
def download_csv(meta): def download_csv(meta):
exp_id = meta["exp_id"] exp_id = meta["exp_id"]
data = generate_csv(exp_id)
# error handling
if isinstance(data, Exception):
emit('timeout', {'exc': str(data)})
return
# create temporary file # create temporary file
fd, path = mkstemp() fd, path = mkstemp()
with os.fdopen(fd, 'w') as tmp: with os.fdopen(fd, 'w', buffering=1) as tmp:
tmp.write(data) if generate_csv(exp_id, tmp):
tmp.flush() # return path and filename to front so user can start downloading
filename = "experiment_{}_{}".format(
# return path and filename to front so user can start downloading exp_id, date.today().strftime("%Y-%m-%d"))
filename = "experiment_{}_{}".format( path = path.split('/')[-1]
exp_id, date.today().strftime("%Y-%m-%d")) emit('file_ready', {'path': path, 'filename': filename})
path = path.split('/')[-1] else:
emit('file_ready', {'path': path, 'filename': filename}) emit('timeout', {'exc': 'job failed'})
@socketio.on('end', namespace="/download_csv") @socketio.on('end', namespace="/download_csv")
......
...@@ -109,7 +109,7 @@ def map_answers_to_questions(answers, questions): ...@@ -109,7 +109,7 @@ def map_answers_to_questions(answers, questions):
@timeit @timeit
def generate_csv(exp_id): def generate_csv(exp_id, file_handle):
# answer sets with participant ids # answer sets with participant ids
participants = answer_set.query.filter_by( participants = answer_set.query.filter_by(
...@@ -129,8 +129,6 @@ def generate_csv(exp_id): ...@@ -129,8 +129,6 @@ def generate_csv(exp_id):
embody_questions = embody_question.query.filter_by( embody_questions = embody_question.query.filter_by(
experiment_idexperiment=exp_id).all() experiment_idexperiment=exp_id).all()
csv = ''
# create CSV-header # create CSV-header
header = 'participant id;' header = 'participant id;'
header += ';'.join([str(count) + '. bg_question: ' + q.background_question.strip() header += ';'.join([str(count) + '. bg_question: ' + q.background_question.strip()
...@@ -146,7 +144,7 @@ def generate_csv(exp_id): ...@@ -146,7 +144,7 @@ def generate_csv(exp_id):
header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' + header += ';' + ';'.join(['page' + str(idx) + '_' + str(count) + '. embody_question: ' +
question.picture.strip() for count, question in enumerate(embody_questions, 1)]) question.picture.strip() for count, question in enumerate(embody_questions, 1)])
csv += header + '\r\n' file_handle.write(header + '\r\n')
# filter empty answer_sets # filter empty answer_sets
participants = list(filter(lambda participant: True if int( participants = list(filter(lambda participant: True if int(
...@@ -167,16 +165,17 @@ def generate_csv(exp_id): ...@@ -167,16 +165,17 @@ def generate_csv(exp_id):
try: try:
emit('progress', {'done': nth, 'from': len_participants}) emit('progress', {'done': nth, 'from': len_participants})
data = future.result() data = future.result()
csv += data + '\r\n' file_handle.write(data + '\n')
# to ensure that all internal buffers associated with fd are written to disk
file_handle.flush()
except Exception as exc: except Exception as exc:
print('generated an exception: {}'.format(exc)) print('generated an exception: {}'.format(exc))
return exc # return False
return csv return True
def generate_answer_row(participant, pages, questions, embody_questions): def generate_answer_row(participant, pages, questions, embody_questions):
# TODO: refactor
with app.app_context(): with app.app_context():
...@@ -264,7 +263,8 @@ def generate_answer_row(participant, pages, questions, embody_questions): ...@@ -264,7 +263,8 @@ def generate_answer_row(participant, pages, questions, embody_questions):
answers_list.append(json.dumps(coordinates_to_bitmap)) answers_list.append(json.dumps(coordinates_to_bitmap))
except ValueError as err: except ValueError as err:
app.logger(err) print(err)
#app.logger(err)
answer_row += ';'.join(answers_list) if embody_answers else \ answer_row += ';'.join(answers_list) if embody_answers else \
len(embody_questions) * len(pages) * ';' len(embody_questions) * len(pages) * ';'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment