diff options
Diffstat (limited to 'src/daq/accesslog/sfive-accesslog.py')
-rwxr-xr-x | src/daq/accesslog/sfive-accesslog.py | 60 |
1 files changed, 38 insertions, 22 deletions
diff --git a/src/daq/accesslog/sfive-accesslog.py b/src/daq/accesslog/sfive-accesslog.py index 3d45d61..1914820 100755 --- a/src/daq/accesslog/sfive-accesslog.py +++ b/src/daq/accesslog/sfive-accesslog.py @@ -94,6 +94,37 @@ class SFiveProto(protocol.ConnectedDatagramProtocol): self._importer._socketError() +class ClientList: + + def __init__(self, file_re): + self._clients = { } + self._file_re = file_re + + def clear(self): + self._clients = { } + + def getCnt(self): + return len(self._clients) + + def getBytesSent(self): + return sum(self._clients.itervalues()) + + def update(self, linedata): + if linedata['status'] != 200 and linedata['status'] != 206: + return + if linedata['req']['cmd'] != 'GET': + return + + try: + if re.match(self._file_re, linedata['req']['url']): + if linedata['client'] in self._clients.keys(): + self._clients[linedata['client']] += linedata['size'] + else: + self._clients[linedata['client']] = linedata['size'] + except re.error as e: + print 'SFive: regex error: %s' % (e) + + class AccessLog(): """Class to batch import nginx/apache access logs into the spreadspace streaming statistic suite""" @@ -210,25 +241,10 @@ class AccessLog(): linedata['ts'] = self._parseDatetime(linedata['ts']) return linedata - def _updateClients(self, clients, linedata): - if linedata['status'] != 200 and linedata['status'] != 206: - return - if linedata['req']['cmd'] != 'GET': - return - - try: - if re.match(self._file_re, linedata['req']['url']): - if linedata['client'] in clients.keys(): - clients[linedata['client']] += linedata['size'] - else: - clients[linedata['client']] = linedata['size'] - except re.error as e: - print 'SFive: regex error: %s' % (e) - def _sendLogData(self, data, lastts): cnt = 0 nextts = None if not lastts else lastts + datetime.timedelta(seconds=self._duration) - clients = { } + clients = ClientList(self._file_re) try: regex = self._prepareLineRegex() for line in data: @@ -236,21 +252,21 @@ class AccessLog(): if not lastts: lastts = linedata['ts'] nextts = lastts + datetime.timedelta(seconds=self._duration) - clients = { } - self._updateClients(clients, linedata) + clients.clear() + clients.update(linedata) else: while linedata['ts'] > nextts: - self._sendDataset(nextts, self._duration, len(clients), sum(clients.itervalues())) + self._sendDataset(nextts, self._duration, clients.getCnt(), clients.getBytesSent()) cnt += 1 lastts = nextts nextts = lastts + datetime.timedelta(seconds=self._duration) - clients = { } + clients.clear() - self._updateClients(clients, linedata) + clients.update(linedata) # send remaining data if nextts: - self._sendDataset(nextts, self._duration, len(clients), sum(clients.itervalues())) + self._sendDataset(nextts, self._duration, clients.getCnt(), clients.getBytesSent()) cnt += 1 except re.error as e: |