From ebd66dc169799e6d29783da9448ff429c6a30c4f Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 24 Apr 2017 00:22:03 +0200 Subject: store user agents in a seperate bucket --- src/hub/src/spreadspace.org/sfive/s5store.go | 78 ++++++++++++++++++++--- src/hub/src/spreadspace.org/sfive/s5typesApi.go | 6 +- src/hub/src/spreadspace.org/sfive/s5typesStore.go | 14 +++- 3 files changed, 85 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/hub/src/spreadspace.org/sfive/s5store.go b/src/hub/src/spreadspace.org/sfive/s5store.go index 64424dc..e8f82b5 100644 --- a/src/hub/src/spreadspace.org/sfive/s5store.go +++ b/src/hub/src/spreadspace.org/sfive/s5store.go @@ -73,7 +73,13 @@ func initDbBolt(boltPath string) (boltDb *bolt.DB, err error) { } err = boltDb.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte(clientDataBn)) + if _, err := tx.CreateBucketIfNotExists([]byte(clientDataBn)); err != nil { + return err + } + if _, err := tx.CreateBucketIfNotExists([]byte(userAgentsFwdBn)); err != nil { + return err + } + _, err := tx.CreateBucketIfNotExists([]byte(userAgentsRevBn)) return err }) @@ -277,21 +283,58 @@ func (s sqliteStore) insertDataUpdateEntry(src sourceDb, du *dataUpdateDb) (err return } -// itob returns an 8-byte big endian representation of v. func itob(v int) []byte { b := make([]byte, 8) binary.BigEndian.PutUint64(b, uint64(v)) return b } +func btoi(b []byte) int { + return int(binary.BigEndian.Uint64(b)) +} + +func (s sqliteStore) insertNewUserAgent(tx *bolt.Tx, ua string) (uaId int, err error) { + bf := tx.Bucket([]byte(userAgentsFwdBn)) + bf.FillPercent = 1.0 // we only do appends + br := tx.Bucket([]byte(userAgentsRevBn)) + br.FillPercent = 1.0 // we only do appends + + bUaId := bf.Get([]byte(ua)) + if bUaId != nil { + return btoi(bUaId), nil + } + + next, _ := bf.NextSequence() + uaId = int(next) + if err = bf.Put([]byte(ua), itob(uaId)); err != nil { + return + } + if err = br.Put(itob(uaId), []byte(ua)); err != nil { + return + } + + return uaId, err +} + func (s sqliteStore) insertDataUpdateClientEntries(cd []ClientData, du dataUpdateDb) error { if len(cd) == 0 { return nil } + return s.dbBolt.Update(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(clientDataBn)) - b.FillPercent = 1.0 // we only do append - jsonData, err := json.Marshal(cd) + b.FillPercent = 1.0 // we only do appends + + data := []clientDataDb{} + for _, c := range cd { + uaId, err := s.insertNewUserAgent(tx, c.UserAgent) + if err != nil { + return err + } + data = append(data, clientDataDb{c.Ip, uaId, c.BytesSent}) + } + + jsonData, err := json.Marshal(data) if err != nil { return err } @@ -401,14 +444,27 @@ func (s sqliteStore) GetUpdate(id int) (res dataUpdateDb, err error) { } func (s sqliteStore) GetClientsByUpdateId(id int) (res []ClientData, err error) { - err = s.dbBolt.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte(clientDataBn)) + err = s.dbBolt.View(func(tx *bolt.Tx) error { + bc := tx.Bucket([]byte(clientDataBn)) + bu := tx.Bucket([]byte(userAgentsRevBn)) - jsonData := b.Get(itob(id)) + jsonData := bc.Get(itob(id)) if jsonData == nil { return nil } - return json.Unmarshal(jsonData, &res) + data := []clientDataDb{} + if err := json.Unmarshal(jsonData, &data); err != nil { + return err + } + for _, c := range data { + cd := ClientData{Ip: c.Ip, BytesSent: c.BytesSent} + ua := bu.Get(itob(c.UserAgentId)) + if ua != nil { + cd.UserAgent = string(ua) + } + res = append(res, cd) + } + return nil }) return } @@ -475,6 +531,12 @@ func (s sqliteStore) GetUpdatesAfter(id int) (res []StatisticsData, err error) { } func (s sqliteStore) GetUpdates(filter *StatsFilter) (res []StatisticsData, err error) { + limit := 5000 + if filter.limit == nil { + filter.limit = &limit + } else if *filter.limit > limit { + *filter.limit = limit + } sourceSql, parameters := getFilteredDataUpdateSelect(filter) sql := "SELECT " + updateColumnSelect + " FROM " + sourceSql s5tl.Printf("store: sql: %s", sql) diff --git a/src/hub/src/spreadspace.org/sfive/s5typesApi.go b/src/hub/src/spreadspace.org/sfive/s5typesApi.go index 5b2b29f..515b869 100644 --- a/src/hub/src/spreadspace.org/sfive/s5typesApi.go +++ b/src/hub/src/spreadspace.org/sfive/s5typesApi.go @@ -23,15 +23,15 @@ type SourceId struct { type ClientData struct { Ip string `json:"ip"` - UserAgent string `json:"user-agent"` + UserAgent string `json:"user-agent,omitempty"` BytesSent uint `json:"bytes-sent"` } type SourceData struct { ClientCount uint `json:"client-count"` - BytesReceived uint `json:"bytes-received"` + BytesReceived uint `json:"bytes-received,omitempty"` BytesSent uint `json:"bytes-sent"` - Clients []ClientData `json:"clients"` + Clients []ClientData `json:"clients,omitempty"` } type DataUpdate struct { diff --git a/src/hub/src/spreadspace.org/sfive/s5typesStore.go b/src/hub/src/spreadspace.org/sfive/s5typesStore.go index 19c9404..a177aea 100644 --- a/src/hub/src/spreadspace.org/sfive/s5typesStore.go +++ b/src/hub/src/spreadspace.org/sfive/s5typesStore.go @@ -7,15 +7,18 @@ import ( // compared to JSON DTOs, DB types are flattened, and use key-relations instead of collections // this is very much not normalized at all, because I'm too lazy to type -// table names const ( + // sqlite table names tagsTn = "Tags" sourceTagsTn = "StreamToTagMap" sourcesTn = "Sources" dataUpdatesTn = "DataUpdates" hubInfoTn = "HubInfo" - clientDataBn = "ClientData" + // bolt bucket names + clientDataBn = "ClientData" + userAgentsFwdBn = "UserAgentsFwd" + userAgentsRevBn = "UserAgentsRev" ) type hubInfoDb struct { @@ -43,6 +46,13 @@ type sourceDb struct { SourceId } +// stored in clientDataBn +type clientDataDb struct { + Ip string `json:"ip"` + UserAgentId int `json:"ua"` + BytesSent uint `json:"bs"` +} + // stored in dataUpdatesTn // in DB, StatisticsData/DataUpdate is flattened compared to JSON DTOs type dataUpdateDb struct { -- cgit v1.2.3