summaryrefslogtreecommitdiff
path: root/src/daq/accesslog/sfive-accesslog.py
diff options
context:
space:
mode:
authorChristian Pointner <equinox@spreadspace.org>2014-10-14 01:45:54 +0200
committerChristian Pointner <equinox@spreadspace.org>2014-10-14 01:45:54 +0200
commit74e72d7e2c736c57e73c5941e13e30a0dfc41c47 (patch)
treefac118c3b8dc95417b8d287c1a4866ec40d0fc9f /src/daq/accesslog/sfive-accesslog.py
parentaggregated stats work now (diff)
moved client hash to own class
Diffstat (limited to 'src/daq/accesslog/sfive-accesslog.py')
-rwxr-xr-xsrc/daq/accesslog/sfive-accesslog.py60
1 files changed, 38 insertions, 22 deletions
diff --git a/src/daq/accesslog/sfive-accesslog.py b/src/daq/accesslog/sfive-accesslog.py
index 3d45d61..1914820 100755
--- a/src/daq/accesslog/sfive-accesslog.py
+++ b/src/daq/accesslog/sfive-accesslog.py
@@ -94,6 +94,37 @@ class SFiveProto(protocol.ConnectedDatagramProtocol):
self._importer._socketError()
+class ClientList:
+
+ def __init__(self, file_re):
+ self._clients = { }
+ self._file_re = file_re
+
+ def clear(self):
+ self._clients = { }
+
+ def getCnt(self):
+ return len(self._clients)
+
+ def getBytesSent(self):
+ return sum(self._clients.itervalues())
+
+ def update(self, linedata):
+ if linedata['status'] != 200 and linedata['status'] != 206:
+ return
+ if linedata['req']['cmd'] != 'GET':
+ return
+
+ try:
+ if re.match(self._file_re, linedata['req']['url']):
+ if linedata['client'] in self._clients.keys():
+ self._clients[linedata['client']] += linedata['size']
+ else:
+ self._clients[linedata['client']] = linedata['size']
+ except re.error as e:
+ print 'SFive: regex error: %s' % (e)
+
+
class AccessLog():
"""Class to batch import nginx/apache access logs into the spreadspace streaming statistic suite"""
@@ -210,25 +241,10 @@ class AccessLog():
linedata['ts'] = self._parseDatetime(linedata['ts'])
return linedata
- def _updateClients(self, clients, linedata):
- if linedata['status'] != 200 and linedata['status'] != 206:
- return
- if linedata['req']['cmd'] != 'GET':
- return
-
- try:
- if re.match(self._file_re, linedata['req']['url']):
- if linedata['client'] in clients.keys():
- clients[linedata['client']] += linedata['size']
- else:
- clients[linedata['client']] = linedata['size']
- except re.error as e:
- print 'SFive: regex error: %s' % (e)
-
def _sendLogData(self, data, lastts):
cnt = 0
nextts = None if not lastts else lastts + datetime.timedelta(seconds=self._duration)
- clients = { }
+ clients = ClientList(self._file_re)
try:
regex = self._prepareLineRegex()
for line in data:
@@ -236,21 +252,21 @@ class AccessLog():
if not lastts:
lastts = linedata['ts']
nextts = lastts + datetime.timedelta(seconds=self._duration)
- clients = { }
- self._updateClients(clients, linedata)
+ clients.clear()
+ clients.update(linedata)
else:
while linedata['ts'] > nextts:
- self._sendDataset(nextts, self._duration, len(clients), sum(clients.itervalues()))
+ self._sendDataset(nextts, self._duration, clients.getCnt(), clients.getBytesSent())
cnt += 1
lastts = nextts
nextts = lastts + datetime.timedelta(seconds=self._duration)
- clients = { }
+ clients.clear()
- self._updateClients(clients, linedata)
+ clients.update(linedata)
# send remaining data
if nextts:
- self._sendDataset(nextts, self._duration, len(clients), sum(clients.itervalues()))
+ self._sendDataset(nextts, self._duration, clients.getCnt(), clients.getBytesSent())
cnt += 1
except re.error as e: