store user agents in a seperate bucket

3 files changed, 85 insertions, 13 deletions

diff --git a/src/hub/src/spreadspace.org/sfive/s5store.go b/src/hub/src/spreadspace.org/sfive/s5store.go
index 64424dc..e8f82b5 100644
--- a/src/hub/src/spreadspace.org/sfive/s5store.go
+++ b/src/hub/src/spreadspace.org/sfive/s5store.go
@@ -73,7 +73,13 @@ func initDbBolt(boltPath string) (boltDb *bolt.DB, err error) {
 	}
 
 	err = boltDb.Update(func(tx *bolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(clientDataBn))
+		if _, err := tx.CreateBucketIfNotExists([]byte(clientDataBn)); err != nil {
+			return err
+		}
+		if _, err := tx.CreateBucketIfNotExists([]byte(userAgentsFwdBn)); err != nil {
+			return err
+		}
+		_, err := tx.CreateBucketIfNotExists([]byte(userAgentsRevBn))
 		return err
 	})
 
@@ -277,21 +283,58 @@ func (s sqliteStore) insertDataUpdateEntry(src sourceDb, du *dataUpdateDb) (err
 	return
 }
 
-// itob returns an 8-byte big endian representation of v.
 func itob(v int) []byte {
 	b := make([]byte, 8)
 	binary.BigEndian.PutUint64(b, uint64(v))
 	return b
 }
 
+func btoi(b []byte) int {
+	return int(binary.BigEndian.Uint64(b))
+}
+
+func (s sqliteStore) insertNewUserAgent(tx *bolt.Tx, ua string) (uaId int, err error) {
+	bf := tx.Bucket([]byte(userAgentsFwdBn))
+	bf.FillPercent = 1.0 // we only do appends
+	br := tx.Bucket([]byte(userAgentsRevBn))
+	br.FillPercent = 1.0 // we only do appends
+
+	bUaId := bf.Get([]byte(ua))
+	if bUaId != nil {
+		return btoi(bUaId), nil
+	}
+
+	next, _ := bf.NextSequence()
+	uaId = int(next)
+	if err = bf.Put([]byte(ua), itob(uaId)); err != nil {
+		return
+	}
+	if err = br.Put(itob(uaId), []byte(ua)); err != nil {
+		return
+	}
+
+	return uaId, err
+}
+
 func (s sqliteStore) insertDataUpdateClientEntries(cd []ClientData, du dataUpdateDb) error {
 	if len(cd) == 0 {
 		return nil
 	}
+
 	return s.dbBolt.Update(func(tx *bolt.Tx) error {
 		b := tx.Bucket([]byte(clientDataBn))
-		b.FillPercent = 1.0 // we only do append
-		jsonData, err := json.Marshal(cd)
+		b.FillPercent = 1.0 // we only do appends
+
+		data := []clientDataDb{}
+		for _, c := range cd {
+			uaId, err := s.insertNewUserAgent(tx, c.UserAgent)
+			if err != nil {
+				return err
+			}
+			data = append(data, clientDataDb{c.Ip, uaId, c.BytesSent})
+		}
+
+		jsonData, err := json.Marshal(data)
 		if err != nil {
 			return err
 		}
@@ -401,14 +444,27 @@ func (s sqliteStore) GetUpdate(id int) (res dataUpdateDb, err error) {
 }
 
 func (s sqliteStore) GetClientsByUpdateId(id int) (res []ClientData, err error) {
-	err = s.dbBolt.Update(func(tx *bolt.Tx) error {
-		b := tx.Bucket([]byte(clientDataBn))
+	err = s.dbBolt.View(func(tx *bolt.Tx) error {
+		bc := tx.Bucket([]byte(clientDataBn))
+		bu := tx.Bucket([]byte(userAgentsRevBn))
 
-		jsonData := b.Get(itob(id))
+		jsonData := bc.Get(itob(id))
 		if jsonData == nil {
 			return nil
 		}
-		return json.Unmarshal(jsonData, &res)
+		data := []clientDataDb{}
+		if err := json.Unmarshal(jsonData, &data); err != nil {
+			return err
+		}
+		for _, c := range data {
+			cd := ClientData{Ip: c.Ip, BytesSent: c.BytesSent}
+			ua := bu.Get(itob(c.UserAgentId))
+			if ua != nil {
+				cd.UserAgent = string(ua)
+			}
+			res = append(res, cd)
+		}
+		return nil
 	})
 	return
 }
@@ -475,6 +531,12 @@ func (s sqliteStore) GetUpdatesAfter(id int) (res []StatisticsData, err error) {
 }
 
 func (s sqliteStore) GetUpdates(filter *StatsFilter) (res []StatisticsData, err error) {
+	limit := 5000
+	if filter.limit == nil {
+		filter.limit = &limit
+	} else if *filter.limit > limit {
+		*filter.limit = limit
+	}
 	sourceSql, parameters := getFilteredDataUpdateSelect(filter)
 	sql := "SELECT " + updateColumnSelect + " FROM " + sourceSql
 	s5tl.Printf("store: sql: %s", sql)
diff --git a/src/hub/src/spreadspace.org/sfive/s5typesApi.go b/src/hub/src/spreadspace.org/sfive/s5typesApi.go
index 5b2b29f..515b869 100644
--- a/src/hub/src/spreadspace.org/sfive/s5typesApi.go
+++ b/src/hub/src/spreadspace.org/sfive/s5typesApi.go
@@ -23,15 +23,15 @@ type SourceId struct {
 
 type ClientData struct {
 	Ip        string `json:"ip"`
-	UserAgent string `json:"user-agent"`
+	UserAgent string `json:"user-agent,omitempty"`
 	BytesSent uint   `json:"bytes-sent"`
 }
 
 type SourceData struct {
 	ClientCount   uint         `json:"client-count"`
-	BytesReceived uint         `json:"bytes-received"`
+	BytesReceived uint         `json:"bytes-received,omitempty"`
 	BytesSent     uint         `json:"bytes-sent"`
-	Clients       []ClientData `json:"clients"`
+	Clients       []ClientData `json:"clients,omitempty"`
 }
 
 type DataUpdate struct {
diff --git a/src/hub/src/spreadspace.org/sfive/s5typesStore.go b/src/hub/src/spreadspace.org/sfive/s5typesStore.go
index 19c9404..a177aea 100644
--- a/src/hub/src/spreadspace.org/sfive/s5typesStore.go
+++ b/src/hub/src/spreadspace.org/sfive/s5typesStore.go
@@ -7,15 +7,18 @@ import (
 // compared to JSON DTOs, DB types are flattened, and use key-relations instead of collections
 // this is very much not normalized at all, because I'm too lazy to type
 
-// table names
 const (
+	// sqlite table names
 	tagsTn        = "Tags"
 	sourceTagsTn  = "StreamToTagMap"
 	sourcesTn     = "Sources"
 	dataUpdatesTn = "DataUpdates"
 	hubInfoTn     = "HubInfo"
 
-	clientDataBn = "ClientData"
+	// bolt bucket names
+	clientDataBn    = "ClientData"
+	userAgentsFwdBn = "UserAgentsFwd"
+	userAgentsRevBn = "UserAgentsRev"
 )
 
 type hubInfoDb struct {
@@ -43,6 +46,13 @@ type sourceDb struct {
 	SourceId
 }
 
+// stored in clientDataBn
+type clientDataDb struct {
+	Ip          string `json:"ip"`
+	UserAgentId int    `json:"ua"`
+	BytesSent   uint   `json:"bs"`
+}
+
 // stored in dataUpdatesTn
 // in DB, StatisticsData/DataUpdate is flattened compared to JSON DTOs
 type dataUpdateDb struct {