Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Timo Heikkilä
PET-rating
Commits
9968f630
Commit
9968f630
authored
Jan 26, 2021
by
Ossi Laine
Browse files
Merge branch 'feat-optimize-data-export' into 'master'
Feat optimize data export See merge request tithei/pet-rating!8
parents
1311884b
c3aab919
Changes
2
Hide whitespace changes
Inline
Side-by-side
app/experiment/views.py
View file @
9968f630
...
...
@@ -1062,24 +1062,17 @@ def start_download_csv():
def
download_csv
(
meta
):
exp_id
=
meta
[
"exp_id"
]
data
=
generate_csv
(
exp_id
)
# error handling
if
isinstance
(
data
,
Exception
):
emit
(
'timeout'
,
{
'exc'
:
str
(
data
)})
return
# create temporary file
fd
,
path
=
mkstemp
()
with
os
.
fdopen
(
fd
,
'w'
)
as
tmp
:
tmp
.
write
(
data
)
tmp
.
flush
()
# return path and filename to front so user can start downloading
filename
=
"experiment_{}_{}"
.
format
(
exp_id
,
date
.
today
().
strftime
(
"%Y-%m-%d"
)
)
path
=
path
.
split
(
'/'
)[
-
1
]
emit
(
'
file_ready'
,
{
'path'
:
path
,
'filename'
:
filename
})
with
os
.
fdopen
(
fd
,
'w'
,
buffering
=
1
)
as
tmp
:
if
generate_csv
(
exp_id
,
tmp
):
# return path and filename to front so user can start downloading
filename
=
"experiment_{}_{}"
.
format
(
exp_id
,
date
.
today
().
strftime
(
"%Y-%m-%d"
))
path
=
path
.
split
(
'/'
)[
-
1
]
emit
(
'file_ready'
,
{
'path'
:
path
,
'filename'
:
filename
}
)
else
:
emit
(
'
timeout'
,
{
'exc'
:
'job failed'
})
@
socketio
.
on
(
'end'
,
namespace
=
"/download_csv"
)
...
...
app/utils.py
View file @
9968f630
...
...
@@ -109,7 +109,7 @@ def map_answers_to_questions(answers, questions):
@
timeit
def
generate_csv
(
exp_id
):
def
generate_csv
(
exp_id
,
file_handle
):
# answer sets with participant ids
participants
=
answer_set
.
query
.
filter_by
(
...
...
@@ -129,8 +129,6 @@ def generate_csv(exp_id):
embody_questions
=
embody_question
.
query
.
filter_by
(
experiment_idexperiment
=
exp_id
).
all
()
csv
=
''
# create CSV-header
header
=
'participant id;'
header
+=
';'
.
join
([
str
(
count
)
+
'. bg_question: '
+
q
.
background_question
.
strip
()
...
...
@@ -146,7 +144,7 @@ def generate_csv(exp_id):
header
+=
';'
+
';'
.
join
([
'page'
+
str
(
idx
)
+
'_'
+
str
(
count
)
+
'. embody_question: '
+
question
.
picture
.
strip
()
for
count
,
question
in
enumerate
(
embody_questions
,
1
)])
csv
+=
header
+
'
\r\n
'
file_handle
.
write
(
header
+
'
\r\n
'
)
# filter empty answer_sets
participants
=
list
(
filter
(
lambda
participant
:
True
if
int
(
...
...
@@ -167,16 +165,17 @@ def generate_csv(exp_id):
try
:
emit
(
'progress'
,
{
'done'
:
nth
,
'from'
:
len_participants
})
data
=
future
.
result
()
csv
+=
data
+
'
\r\n
'
file_handle
.
write
(
data
+
'
\n
'
)
# to ensure that all internal buffers associated with fd are written to disk
file_handle
.
flush
()
except
Exception
as
exc
:
print
(
'generated an exception: {}'
.
format
(
exc
))
return
exc
return
csv
#
return
False
return
True
def
generate_answer_row
(
participant
,
pages
,
questions
,
embody_questions
):
# TODO: refactor
with
app
.
app_context
():
...
...
@@ -264,7 +263,8 @@ def generate_answer_row(participant, pages, questions, embody_questions):
answers_list
.
append
(
json
.
dumps
(
coordinates_to_bitmap
))
except
ValueError
as
err
:
app
.
logger
(
err
)
print
(
err
)
#app.logger(err)
answer_row
+=
';'
.
join
(
answers_list
)
if
embody_answers
else
\
len
(
embody_questions
)
*
len
(
pages
)
*
';'
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment