diff --git a/contrib/src/web/database.py b/contrib/src/web/database.py new file mode 100644 index 0000000000000000000000000000000000000000..9f2bab047cc750d35f3e65d66752312bbecb2b54 --- /dev/null +++ b/contrib/src/web/database.py @@ -0,0 +1,83 @@ +#!/usr/bin/python +# coding=utf-8 + +import sys + +from peewee import Model, CharField, IntegerField, DateTimeField, PostgresqlDatabase, CompositeKey +from web.faces import getCurrentTime + +# leading underscore, denoting module private variable, may exclude this variable from wildcard imports on some versions of python +_database = PostgresqlDatabase('stop', user='stop', password='PT52lRecp4NBKQrZT9', host='database') + +# PostgreSQL table classes: +class UnknownField(object): + def __init__(self, *_, **__): pass + +class BaseModel(Model): + class Meta: + database = _database + +class GenderStats(BaseModel): + rowid = IntegerField() + t = CharField(column_name='t_id', max_length=10) + t_ts = DateTimeField(default=getCurrentTime) + gender_text = CharField(null=True, max_length=100) + + class Meta: + table_name = 'gender_stats' + indexes = ( + (('t', 't_ts', 'rowid'), True), + ) + primary_key = CompositeKey('rowid', 't', 't_ts') + +def truncateString(str, size): + if size <= 3: + return str + + return (str[:(size-3)] + '...') if len(str) > size else str + +def connect(): + if _database.is_closed(): + _database.connect() + return _database + +def close(db): + if not db.is_closed(): + db.close() + +def addStats(data, stopCode = None): + stopCode = stopCode if stopCode else 'unknown' + + with _database.atomic(): + i = 0 + ts = getCurrentTime() # only one value for current time per request + for face in data: + if not face['gender']: + face['gender'] = 'unknown' + + GenderStats.create( + t = truncateString(stopCode, GenderStats.t.max_length), + t_ts = ts, + rowid = i, + gender_text = truncateString(face['gender'], GenderStats.gender_text.max_length) + ) + + # NOTE: having serial id here has low risk of PK collision iff parallel processing of requests is used. + # Can be done f.ex. through multiple uwsgi workers, which the current setup does not use + i += 1 + +def main(argv): + db = connect() + + # create database tables + if not GenderStats.table_exists(): + GenderStats.create_table() + print("Created table: {0}".format(GenderStats.Meta.table_name)) + + print("Database successfully initalized...") + close(db) + + return True + +if __name__ == "__main__": + main(sys.argv) diff --git a/contrib/src/web/faces.py b/contrib/src/web/faces.py index 15f969401ef994030d226e903f6719a8decd6937..b455ae1694c83d8c13a6e4760efd33ec1de6c940 100644 --- a/contrib/src/web/faces.py +++ b/contrib/src/web/faces.py @@ -5,8 +5,8 @@ import logging, requests, os, weakref from flask import Flask, jsonify, make_response, request, abort, redirect, send_file from flask_request_id import RequestID from web.emotion_gender_processor import EGProcessor +from web import database from flask import g -from peewee import Model, CharField, IntegerField, DateTimeField, PostgresqlDatabase, CompositeKey from datetime import datetime from dateutil.tz import tzlocal @@ -16,9 +16,6 @@ if not os.path.exists(dirname): logging.basicConfig(filename = os.path.join(dirname, 'debug.log'), level = logging.DEBUG) -# PostgreSQL table classes: -database = PostgresqlDatabase('stop', user='stop', password='PT52lRecp4NBKQrZT9', host='database') - # NOTE: most docker images run in UTC by default def getCurrentTime(format = None): ts = datetime.now(tzlocal()) @@ -27,32 +24,6 @@ def getCurrentTime(format = None): return ts.strftime(format) -def truncateString(str, size): - if size <= 3: - return str - - return (str[:(size-3)] + '...') if len(str) > size else str - -class UnknownField(object): - def __init__(self, *_, **__): pass - -class BaseModel(Model): - class Meta: - database = database - -class Gender_Stats(BaseModel): - rowid = IntegerField() - t = CharField(column_name='t_id', max_length=10) - t_ts = DateTimeField(default=getCurrentTime) - gender_text = CharField(null=True, max_length=100) - - class Meta: - table_name = 'Gender_Stats' - indexes = ( - (('t', 't_ts', 'rowid'), True), - ) - primary_key = CompositeKey('rowid', 't', 't_ts') - # File removal after processing (match with legal documentation about storage) class FileCleaner(object): def __init__(self): @@ -81,14 +52,11 @@ cleaner = FileCleaner() @app.before_request def before_request(): - g.db = database - if g.db.is_closed(): - g.db.connect() + g.db = database.connect() @app.teardown_request def teardown_request(exc): - if not g.db.is_closed(): - g.db.close() + database.close(g.db) @app.route('/') def index(): @@ -97,6 +65,9 @@ def index(): @app.route('/api/v<int:version>/classifyImage/', methods=['POST']) @app.route('/api/v<int:version>/classifyImage/<type>', methods=['POST']) def upload(version, type = 'mem'): + response = None + resultFile = None + try: if int(version) != 1: raise Exception('Invalid API version number.') @@ -114,23 +85,7 @@ def upload(version, type = 'mem'): result = processor.processImage(request.files['image'].read(), result_fname = getRequestID(request), type = type, detect_emotion = detectEmotion != 0) # VH Insert result into database - with database.atomic(): - i = 0 - ts = getCurrentTime() # only one value for current time per request - for face in result: - if not face['gender']: - face['gender'] = 'unknown' - - Gender_Stats.create( - t = truncateString(stopCode, Gender_Stats.t.max_length), - t_ts = ts, - rowid = i, - gender_text = truncateString(face['gender'], Gender_Stats.gender_text.max_length) - ) - - # NOTE: having serial id here has low risk of PK collision iff parallel processing of requests is used. - # Can be done f.ex. through multiple uwsgi workers, which the current setup does not use - i += 1 + database.addStats(result, stopCode) # by default avoid all disk I/O by using the result held in memory, explicit type argument will still roundtrip to disk # NOTE: the ordering in json results is not stable (python dictionaries are unordered) @@ -139,9 +94,6 @@ def upload(version, type = 'mem'): response.headers.extend({'Cache-Control': 'no-cache', 'Expires': '0'}) return response - response = None - resultFile = None - if type == 'png': resultFile = os.path.join(dirname, getRequestID(request) + '.png') response = send_file(resultFile, mimetype='image/png') @@ -160,7 +112,7 @@ def upload(version, type = 'mem'): return response except Exception as err: logging.error('An error has occurred whilst processing the file: "{0}", from: {1}'.format(err, stopCode)) - if resultFile: + if resultFile and os.path.isfile(resultFile): os.remove(resultFile) abort(400) @@ -168,6 +120,9 @@ def upload(version, type = 'mem'): @app.route('/api/v<int:version>/classifyImage/', methods=['GET']) @app.route('/api/v<int:version>/classifyImage/<type>', methods=['GET']) def dowload(version, type = 'mem'): + response = None + resultFile = None + try: if int(version) != 1: raise Exception('Invalid API version number.') @@ -184,23 +139,7 @@ def dowload(version, type = 'mem'): result = processor.processImage(requests.get(uri).content, result_fname = getRequestID(request), type = type, detect_emotion = detectEmotion != 0) # VH Insert result into database - with database.atomic(): - i = 0 - ts = getCurrentTime() # only one value for current time per request - for face in result: - if not face['gender']: - face['gender'] = 'unknown' - - Gender_Stats.create( - t = truncateString(stopCode, Gender_Stats.t.max_length), - t_ts = ts, - rowid = i, - gender_text = truncateString(face['gender'], Gender_Stats.gender_text.max_length) - ) - - # NOTE: having serial id here has low risk of PK collision iff parallel processing of requests is used. - # Can be done f.ex. through multiple uwsgi workers, which the current setup does not use - i += 1 + database.addStats(result, stopCode) # by default avoid all disk I/O by using the result held in memory, explicit type argument will still roundtrip to disk # NOTE: the ordering in json results is not stable (python dictionaries are unordered) @@ -209,9 +148,6 @@ def dowload(version, type = 'mem'): response.headers.extend({'Cache-Control': 'no-cache', 'Expires': '0'}) return response - response = None - resultFile = None - if type == 'png': resultFile = os.path.join(dirname, getRequestID(request) + '.png') response = send_file(resultFile, mimetype='image/png') @@ -230,7 +166,7 @@ def dowload(version, type = 'mem'): return response except Exception as err: logging.error('An error has occurred whilst processing the file: "{0}", from: {1}'.format(err, stopCode)) - if resultFile: + if resultFile and os.path.isfile(resultFile): os.remove(resultFile) abort(400)