diff --git a/contrib/src/web/faces.py b/contrib/src/web/faces.py index 7416d1b705199b180f1958ffef71eb5177c1c350..0f3e0ac8f0cca1d95c6c4a999f1aab0184971463 100644 --- a/contrib/src/web/faces.py +++ b/contrib/src/web/faces.py @@ -27,6 +27,9 @@ def getCurrentTime(format = None): return ts.strftime(format) +def truncateString(str, len): + return (str[:8] + '..') if len(str) > 10 else str + class UnknownField(object): def __init__(self, *_, **__): pass @@ -35,10 +38,10 @@ class BaseModel(Model): database = database class Gender_Stats(BaseModel): - gender_text = CharField(null=True) rowid = IntegerField() - t = CharField(column_name='t_id') + t = CharField(column_name='t_id', max_length=10) t_ts = DateTimeField(default=getCurrentTime) + gender_text = CharField(null=True, max_length=100) class Meta: table_name = 'Gender_Stats' @@ -111,7 +114,18 @@ def upload(version, type = 'mem'): i = 0 ts = getCurrentTime() # only one value for current time per request for face in result: - Gender_Stats.create(t = stopCode, t_ts = ts, rowid = i, gender_text = face["gender"]) + if not face['gender']: + face['gender'] = 'unknown' + + Gender_Stats.create( + t = truncateString(stopCode, Gender_Stats.t.max_length), + t_ts = ts, + rowid = i, + gender_text = truncateString(face['gender'], Gender_Stats.gender_text.max_length) + ) + + # NOTE: having serial id here has low risk of PK collision iff parallel processing of requests is used. + # Can be done f.ex. through multiple uwsgi workers, which the current setup does not use i += 1 # by default avoid all disk I/O by using the result held in memory, explicit type argument will still roundtrip to disk @@ -170,7 +184,18 @@ def dowload(version, type = 'mem'): i = 0 ts = getCurrentTime() # only one value for current time per request for face in result: - Gender_Stats.create(t = stopCode, t_ts = ts, rowid = i, gender_text = face["gender"]) + if not face['gender']: + face['gender'] = 'unknown' + + Gender_Stats.create( + t = truncateString(stopCode, Gender_Stats.t.max_length), + t_ts = ts, + rowid = i, + gender_text = truncateString(face['gender'], Gender_Stats.gender_text.max_length) + ) + + # NOTE: having serial id here has low risk of PK collision iff parallel processing of requests is used. + # Can be done f.ex. through multiple uwsgi workers, which the current setup does not use i += 1 # by default avoid all disk I/O by using the result held in memory, explicit type argument will still roundtrip to disk