Flush writes between every row when generating big CSV export file

app/experiment/views.py

@@ -1062,25 +1062,17 @@ def start_download_csv():
 def download_csv(meta):
     exp_id = meta["exp_id"]
 
-
-    # error handling
-    '''
-    if isinstance(data, Exception):
-        emit('timeout', {'exc': str(data)})
-        return
-    '''
-
     # create temporary file
     fd, path = mkstemp()
-    with os.fdopen(fd, 'w') as tmp:
-
+    with os.fdopen(fd, 'w', buffering=1) as tmp:
         if generate_csv(exp_id, tmp):
-
             # return path and filename to front so user can start downloading
             filename = "experiment_{}_{}".format(
                 exp_id, date.today().strftime("%Y-%m-%d"))
             path = path.split('/')[-1]
             emit('file_ready', {'path': path, 'filename': filename})
+        else:
+            emit('timeout', {'exc': 'job failed'})
 
 
 @socketio.on('end', namespace="/download_csv")

app/utils.py

@@ -166,12 +166,12 @@ def generate_csv(exp_id, file_handle):
                 emit('progress', {'done': nth, 'from': len_participants})
                 data = future.result()
                 file_handle.write(data + '\n')
+                # to ensure that all internal buffers associated with fd are written to disk
+                file_handle.flush()
             except Exception as exc:
                 print('generated an exception: {}'.format(exc))
-                emit('timeout', {'exc': str(data)})
-                return False
+                # return False
     
-    # file_handle.flush()
     return True
 
 
@@ -263,7 +263,8 @@ def generate_answer_row(participant, pages, questions, embody_questions):
                 answers_list.append(json.dumps(coordinates_to_bitmap))
 
             except ValueError as err:
-                app.logger(err)
+                print(err)
+                #app.logger(err)
 
         answer_row += ';'.join(answers_list) if embody_answers else \
             len(embody_questions) * len(pages) * ';'