truncate string fields

contrib/src/web/faces.py

@@ -27,6 +27,9 @@ def getCurrentTime(format = None):
 
     return ts.strftime(format)
 
+def truncateString(str, len):
+    return (str[:8] + '..') if len(str) > 10 else str
+
 class UnknownField(object):
     def __init__(self, *_, **__): pass
 
@@ -35,10 +38,10 @@ class BaseModel(Model):
         database = database
 
 class Gender_Stats(BaseModel):
-    gender_text = CharField(null=True)
     rowid = IntegerField()
-    t = CharField(column_name='t_id')
+    t = CharField(column_name='t_id', max_length=10)
     t_ts = DateTimeField(default=getCurrentTime)
+    gender_text = CharField(null=True, max_length=100)
 
     class Meta:
         table_name = 'Gender_Stats'
@@ -111,7 +114,18 @@ def upload(version, type = 'mem'):
             i = 0
             ts = getCurrentTime() # only one value for current time per request
             for face in result:
-                Gender_Stats.create(t = stopCode, t_ts = ts, rowid = i, gender_text = face["gender"])
+                if not face['gender']:
+                    face['gender'] = 'unknown'
+
+                Gender_Stats.create(
+                    t = truncateString(stopCode, Gender_Stats.t.max_length),
+                    t_ts = ts,
+                    rowid = i,
+                    gender_text = truncateString(face['gender'], Gender_Stats.gender_text.max_length)
+                )
+
+                # NOTE: having serial id here has low risk of PK collision iff parallel processing of requests is used.
+                # Can be done f.ex. through multiple uwsgi workers, which the current setup does not use
                 i += 1
 
         # by default avoid all disk I/O by using the result held in memory, explicit type argument will still roundtrip to disk
@@ -170,7 +184,18 @@ def dowload(version, type = 'mem'):
             i = 0
             ts = getCurrentTime() # only one value for current time per request
             for face in result:
-                Gender_Stats.create(t = stopCode, t_ts = ts, rowid = i, gender_text = face["gender"])
+                if not face['gender']:
+                    face['gender'] = 'unknown'
+
+                Gender_Stats.create(
+                    t = truncateString(stopCode, Gender_Stats.t.max_length),
+                    t_ts = ts,
+                    rowid = i,
+                    gender_text = truncateString(face['gender'], Gender_Stats.gender_text.max_length)
+                )
+
+                # NOTE: having serial id here has low risk of PK collision iff parallel processing of requests is used.
+                # Can be done f.ex. through multiple uwsgi workers, which the current setup does not use
                 i += 1
 
         # by default avoid all disk I/O by using the result held in memory, explicit type argument will still roundtrip to disk