Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Timo Heikkilä
PET-rating
Commits
c3aab919
Commit
c3aab919
authored
Jan 26, 2021
by
Ossi Laine
Browse files
Flush writes between every row when generating big CSV export file
parent
7bebb074
Changes
2
Hide whitespace changes
Inline
Side-by-side
app/experiment/views.py
View file @
c3aab919
...
...
@@ -1062,25 +1062,17 @@ def start_download_csv():
def
download_csv
(
meta
):
exp_id
=
meta
[
"exp_id"
]
# error handling
'''
if isinstance(data, Exception):
emit('timeout', {'exc': str(data)})
return
'''
# create temporary file
fd
,
path
=
mkstemp
()
with
os
.
fdopen
(
fd
,
'w'
)
as
tmp
:
with
os
.
fdopen
(
fd
,
'w'
,
buffering
=
1
)
as
tmp
:
if
generate_csv
(
exp_id
,
tmp
):
# return path and filename to front so user can start downloading
filename
=
"experiment_{}_{}"
.
format
(
exp_id
,
date
.
today
().
strftime
(
"%Y-%m-%d"
))
path
=
path
.
split
(
'/'
)[
-
1
]
emit
(
'file_ready'
,
{
'path'
:
path
,
'filename'
:
filename
})
else
:
emit
(
'timeout'
,
{
'exc'
:
'job failed'
})
@
socketio
.
on
(
'end'
,
namespace
=
"/download_csv"
)
...
...
app/utils.py
View file @
c3aab919
...
...
@@ -166,12 +166,12 @@ def generate_csv(exp_id, file_handle):
emit
(
'progress'
,
{
'done'
:
nth
,
'from'
:
len_participants
})
data
=
future
.
result
()
file_handle
.
write
(
data
+
'
\n
'
)
# to ensure that all internal buffers associated with fd are written to disk
file_handle
.
flush
()
except
Exception
as
exc
:
print
(
'generated an exception: {}'
.
format
(
exc
))
emit
(
'timeout'
,
{
'exc'
:
str
(
data
)})
return
False
# return False
# file_handle.flush()
return
True
...
...
@@ -263,7 +263,8 @@ def generate_answer_row(participant, pages, questions, embody_questions):
answers_list
.
append
(
json
.
dumps
(
coordinates_to_bitmap
))
except
ValueError
as
err
:
app
.
logger
(
err
)
print
(
err
)
#app.logger(err)
answer_row
+=
';'
.
join
(
answers_list
)
if
embody_answers
else
\
len
(
embody_questions
)
*
len
(
pages
)
*
';'
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment