diff options
author | Christian Pointner <equinox@spreadspace.org> | 2017-12-27 12:54:58 +0100 |
---|---|---|
committer | Christian Pointner <equinox@spreadspace.org> | 2017-12-27 12:54:58 +0100 |
commit | 64b37258526b0b592232ca5ace192c2eee032220 (patch) | |
tree | a51a43e9b374c2ab4e2fbf4236baf484d66dbff0 /src/hub | |
parent | switch from boltdb/bolt to coreos/bbolt (diff) |
use int64 for ids everywhere
Diffstat (limited to 'src/hub')
-rw-r--r-- | src/hub/src/spreadspace.org/sfive/s5srvForwardEs.go | 4 | ||||
-rw-r--r-- | src/hub/src/spreadspace.org/sfive/s5srvForwardSfive.go | 10 | ||||
-rw-r--r-- | src/hub/src/spreadspace.org/sfive/s5srvWeb.go | 12 | ||||
-rw-r--r-- | src/hub/src/spreadspace.org/sfive/s5store.go | 54 | ||||
-rw-r--r-- | src/hub/src/spreadspace.org/sfive/s5store_test.go | 12 | ||||
-rw-r--r-- | src/hub/src/spreadspace.org/sfive/s5typesApi.go | 8 | ||||
-rw-r--r-- | src/hub/src/spreadspace.org/sfive/s5typesStore.go | 20 |
7 files changed, 60 insertions, 60 deletions
diff --git a/src/hub/src/spreadspace.org/sfive/s5srvForwardEs.go b/src/hub/src/spreadspace.org/sfive/s5srvForwardEs.go index fb56cbd..cba1add 100644 --- a/src/hub/src/spreadspace.org/sfive/s5srvForwardEs.go +++ b/src/hub/src/spreadspace.org/sfive/s5srvForwardEs.go @@ -47,7 +47,7 @@ const forwardEsLastUpdateIDJson = `{ "aggregations": { "last-id" : { "max" : { "field": "SourceHubUpdateId" } } } }` -func fwdEsGetLastUpdateID(baseUrl string, client *http.Client, hubUUID string) (lastID int, err error) { +func fwdEsGetLastUpdateID(baseUrl string, client *http.Client, hubUUID string) (lastID int64, err error) { url := baseUrl + "/dataupdate/_search?search_type=count" queryJson := fmt.Sprintf(forwardEsLastUpdateIDJson, hubUUID) @@ -92,7 +92,7 @@ func fwdEsGetLastUpdateID(baseUrl string, client *http.Client, hubUUID string) ( return } tid := idstrcntr.(float64) - lastID = int(tid) + lastID = int64(tid) } return diff --git a/src/hub/src/spreadspace.org/sfive/s5srvForwardSfive.go b/src/hub/src/spreadspace.org/sfive/s5srvForwardSfive.go index d09c500..0e0a7e6 100644 --- a/src/hub/src/spreadspace.org/sfive/s5srvForwardSfive.go +++ b/src/hub/src/spreadspace.org/sfive/s5srvForwardSfive.go @@ -41,8 +41,8 @@ import ( "time" ) -func findMaxID(updates []*UpdateFull) int { - maxID := -1 +func findMaxID(updates []*UpdateFull) int64 { + maxID := int64(-1) for _, value := range updates { if id := value.SourceHubUpdateID; id > maxID { maxID = id @@ -51,7 +51,7 @@ func findMaxID(updates []*UpdateFull) int { return maxID } -func fwdGetLastUpdateID(baseUrl string, client *http.Client, hubUUID string) (lastID int, err error) { +func fwdGetLastUpdateID(baseUrl string, client *http.Client, hubUUID string) (lastID int64, err error) { lastID = -1 var resp *http.Response @@ -93,7 +93,7 @@ func fwdWriteUpdates(updates []*UpdateFull, pw *io.PipeWriter) { } } -func fwdPostUpdates(client *http.Client, url string, pr *io.PipeReader) (int, error) { +func fwdPostUpdates(client *http.Client, url string, pr *io.PipeReader) (int64, error) { resp, err := client.Post(url, "application/json", pr) if err != nil { return 0, err @@ -148,7 +148,7 @@ tryResync: if num, err := fwdPostUpdates(client, url, pr); err != nil { s5l.Printf("srv|fwd: sending updates failed: %v", err) continue tryResync - } else if num != len(updates) { + } else if num != int64(len(updates)) { s5l.Printf("srv|fwd: server acknowledged wrong number of updates: expected %d, got: %d", len(updates), num) continue tryResync } diff --git a/src/hub/src/spreadspace.org/sfive/s5srvWeb.go b/src/hub/src/spreadspace.org/sfive/s5srvWeb.go index 7094dc4..7dff6e7 100644 --- a/src/hub/src/spreadspace.org/sfive/s5srvWeb.go +++ b/src/hub/src/spreadspace.org/sfive/s5srvWeb.go @@ -141,7 +141,7 @@ func webUpdatesWithParam(srv *Server, w http.ResponseWriter, r *http.Request) { sendWebResponse(w, http.StatusBadRequest, WebErrorResponse{"invalid update id: " + strings.TrimPrefix("/updates/", r.URL.Path)}) return } - id, err := strconv.Atoi(path.Base(r.URL.Path)) + id, err := strconv.ParseInt(path.Base(r.URL.Path), 10, 64) if err != nil { sendWebResponse(w, http.StatusBadRequest, WebErrorResponse{"invalid update id: " + err.Error()}) return @@ -176,7 +176,7 @@ func webUpdates(srv *Server, w http.ResponseWriter, r *http.Request) { // get one update -func webUpdateGet(srv *Server, id int, w http.ResponseWriter, r *http.Request) { +func webUpdateGet(srv *Server, id int64, w http.ResponseWriter, r *http.Request) { update, err := srv.store.GetUpdate(id) if err != nil { status := http.StatusInternalServerError @@ -275,7 +275,7 @@ func webUpdatesPostBulk(srv *Server, w http.ResponseWriter, r *http.Request) { } updates = append(updates, update) } - numUpdates := len(updates) + numUpdates := int64(len(updates)) if numUpdates < 1 { sendWebResponse(w, http.StatusBadRequest, WebErrorResponse{"got no data messages"}) return @@ -338,12 +338,12 @@ func webLastUpdateID(srv *Server, w http.ResponseWriter, r *http.Request) { // common functions // -func parseAfterAndLimit(r *http.Request) (after, limit int, err error) { +func parseAfterAndLimit(r *http.Request) (after, limit int64, err error) { q := r.URL.Query() afterStr := q.Get("after") if afterStr != "" { - if after, err = strconv.Atoi(afterStr); err != nil { + if after, err = strconv.ParseInt(afterStr, 10, 64); err != nil { err = fmt.Errorf("invalid after value: %v", err.Error()) return } @@ -352,7 +352,7 @@ func parseAfterAndLimit(r *http.Request) (after, limit int, err error) { limit = -1 limitStr := q.Get("limit") if limitStr != "" { - if limit, err = strconv.Atoi(limitStr); err != nil { + if limit, err = strconv.ParseInt(limitStr, 10, 64); err != nil { err = fmt.Errorf("invalid limit value: %v", err.Error()) return } diff --git a/src/hub/src/spreadspace.org/sfive/s5store.go b/src/hub/src/spreadspace.org/sfive/s5store.go index 0615cd2..eb5793c 100644 --- a/src/hub/src/spreadspace.org/sfive/s5store.go +++ b/src/hub/src/spreadspace.org/sfive/s5store.go @@ -54,7 +54,7 @@ var ( ) type Store struct { - version int + version int64 hubUUID string db *bolt.DB readOnly bool @@ -64,7 +64,7 @@ type Store struct { // Initialization and Destruction // -func openDB(dbPath string, readOnly bool) (db *bolt.DB, version int, hubUUID string, err error) { +func openDB(dbPath string, readOnly bool) (db *bolt.DB, version int64, hubUUID string, err error) { if _, err = os.Stat(dbPath); err != nil { if os.IsNotExist(err) { err = nil @@ -112,7 +112,7 @@ func openDB(dbPath string, readOnly bool) (db *bolt.DB, version int, hubUUID str return } -func createDB(dbPath string) (db *bolt.DB, version int, hubUUID string, err error) { +func createDB(dbPath string) (db *bolt.DB, version int64, hubUUID string, err error) { db, err = bolt.Open(dbPath, 0600, &bolt.Options{Timeout: 100 * time.Millisecond}) if err != nil { return @@ -129,7 +129,7 @@ func createDB(dbPath string) (db *bolt.DB, version int, hubUUID string, err erro if err != nil { return err } - version = StoreVersion + version = int64(StoreVersion) if err := b.Put([]byte(storeVersionKey), itob(version)); err != nil { return err } @@ -184,7 +184,7 @@ func (st *Store) Close() { // append key-value pairs to buckets -func (st *Store) insertNewHub(tx *bolt.Tx, hubUUID string) (hubID int, err error) { +func (st *Store) insertNewHub(tx *bolt.Tx, hubUUID string) (hubID int64, err error) { if hubUUID == "" { return } @@ -200,7 +200,7 @@ func (st *Store) insertNewHub(tx *bolt.Tx, hubUUID string) (hubID int, err error } next, _ := bf.NextSequence() - hubID = int(next) + hubID = int64(next) if err = bf.Put([]byte(hubUUID), itob(hubID)); err != nil { return } @@ -211,7 +211,7 @@ func (st *Store) insertNewHub(tx *bolt.Tx, hubUUID string) (hubID int, err error return hubID, err } -func (st *Store) insertNewSource(tx *bolt.Tx, src *sourceDB) (srcID int, err error) { +func (st *Store) insertNewSource(tx *bolt.Tx, src *sourceDB) (srcID int64, err error) { bf := tx.Bucket([]byte(sourcesFwdBn)) // br.FillPercent = 1.0 // these appends are not ordered (the key is the slug and not an integer id) br := tx.Bucket([]byte(sourcesRevBn)) @@ -229,7 +229,7 @@ func (st *Store) insertNewSource(tx *bolt.Tx, src *sourceDB) (srcID int, err err } next, _ := bf.NextSequence() - srcID = int(next) + srcID = int64(next) if err = bf.Put([]byte(slug), itob(srcID)); err != nil { return } @@ -240,12 +240,12 @@ func (st *Store) insertNewSource(tx *bolt.Tx, src *sourceDB) (srcID int, err err return srcID, err } -func (st *Store) insertUpdate(tx *bolt.Tx, du *updateDB) (duID int, err error) { +func (st *Store) insertUpdate(tx *bolt.Tx, du *updateDB) (duID int64, err error) { b := tx.Bucket([]byte(updatesBn)) b.FillPercent = 1.0 // we only do appends next, _ := b.NextSequence() - duID = int(next) + duID = int64(next) var jsonData []byte if jsonData, err = json.Marshal(du); err != nil { @@ -255,7 +255,7 @@ func (st *Store) insertUpdate(tx *bolt.Tx, du *updateDB) (duID int, err error) { return } -func (st *Store) insertNewUserAgent(tx *bolt.Tx, ua string) (uaID int, err error) { +func (st *Store) insertNewUserAgent(tx *bolt.Tx, ua string) (uaID int64, err error) { bf := tx.Bucket([]byte(userAgentsFwdBn)) bf.FillPercent = 1.0 // we only do appends br := tx.Bucket([]byte(userAgentsRevBn)) @@ -267,7 +267,7 @@ func (st *Store) insertNewUserAgent(tx *bolt.Tx, ua string) (uaID int, err error } next, _ := bf.NextSequence() - uaID = int(next) + uaID = int64(next) if err = bf.Put([]byte(ua), itob(uaID)); err != nil { return } @@ -278,7 +278,7 @@ func (st *Store) insertNewUserAgent(tx *bolt.Tx, ua string) (uaID int, err error return uaID, err } -func (st *Store) insertNewClient(tx *bolt.Tx, client *clientDB) (clientID int, err error) { +func (st *Store) insertNewClient(tx *bolt.Tx, client *clientDB) (clientID int64, err error) { bf := tx.Bucket([]byte(clientsFwdBn)) // br.FillPercent = 1.0 // these appends are not ordered (the key is the slug and not an integer id) br := tx.Bucket([]byte(clientsRevBn)) @@ -296,7 +296,7 @@ func (st *Store) insertNewClient(tx *bolt.Tx, client *clientDB) (clientID int, e } next, _ := bf.NextSequence() - clientID = int(next) + clientID = int64(next) if err = bf.Put([]byte(slug), itob(clientID)); err != nil { return } @@ -307,7 +307,7 @@ func (st *Store) insertNewClient(tx *bolt.Tx, client *clientDB) (clientID int, e return clientID, err } -func (st *Store) insertClients(tx *bolt.Tx, uID int, cd []Client) error { +func (st *Store) insertClients(tx *bolt.Tx, uID int64, cd []Client) error { if len(cd) == 0 { return nil } @@ -337,7 +337,7 @@ func (st *Store) insertClients(tx *bolt.Tx, uID int, cd []Client) error { return b.Put(itob(uID), jsonData) } -func (st *Store) setLastUpdateForUUID(tx *bolt.Tx, uuid string, uID int) error { +func (st *Store) setLastUpdateForUUID(tx *bolt.Tx, uuid string, uID int64) error { b := tx.Bucket([]byte(latestUpdatesBn)) b.FillPercent = 1.0 // we only do appends @@ -350,7 +350,7 @@ func (st *Store) setLastUpdateForUUID(tx *bolt.Tx, uuid string, uID int) error { // Split up the multidimensional dataupdate and append all the key-value pairs -func (st *Store) appendItem(tx *bolt.Tx, uf *UpdateFull) (uID int, err error) { +func (st *Store) appendItem(tx *bolt.Tx, uf *UpdateFull) (uID int64, err error) { u := NewUpdateDB(uf) s := NewSourceDB(uf) @@ -402,7 +402,7 @@ func (st *Store) Append(update *UpdateFull) (err error) { // fetch key-value pairs from buckets -func (st *Store) getHub(tx *bolt.Tx, id int) string { +func (st *Store) getHub(tx *bolt.Tx, id int64) string { b := tx.Bucket([]byte(hubUUIDsRevBn)) uuid := b.Get(itob(id)) if uuid == nil { @@ -411,7 +411,7 @@ func (st *Store) getHub(tx *bolt.Tx, id int) string { return string(uuid) } -func (st *Store) getSource(tx *bolt.Tx, id int) (source *sourceDB, err error) { +func (st *Store) getSource(tx *bolt.Tx, id int64) (source *sourceDB, err error) { b := tx.Bucket([]byte(sourcesRevBn)) jsonData := b.Get(itob(id)) @@ -426,7 +426,7 @@ func (st *Store) getSource(tx *bolt.Tx, id int) (source *sourceDB, err error) { return } -func (st *Store) getClients(tx *bolt.Tx, id int) (clients []Client, err error) { +func (st *Store) getClients(tx *bolt.Tx, id int64) (clients []Client, err error) { bcd := tx.Bucket([]byte(clientDataBn)) bc := tx.Bucket([]byte(clientsRevBn)) bu := tx.Bucket([]byte(userAgentsRevBn)) @@ -463,7 +463,7 @@ func (st *Store) getClients(tx *bolt.Tx, id int) (clients []Client, err error) { // fetch all the key-value pairs and merge them into the multidimensional dataupdate -func (st *Store) fetchItem(tx *bolt.Tx, uID int, u *updateDB) (update *UpdateFull, err error) { +func (st *Store) fetchItem(tx *bolt.Tx, uID int64, u *updateDB) (update *UpdateFull, err error) { update = &UpdateFull{} update.CopyFromUpdateDB(u, st.getHub(tx, u.SourceHubID), st.hubUUID, uID) var src *sourceDB @@ -479,7 +479,7 @@ func (st *Store) fetchItem(tx *bolt.Tx, uID int, u *updateDB) (update *UpdateFul // Public Fetch Interface -func (st *Store) GetUpdatesAfter(id, limit int) (updates []*UpdateFull, err error) { +func (st *Store) GetUpdatesAfter(id, limit int64) (updates []*UpdateFull, err error) { updates = []*UpdateFull{} if id < 0 { // TODO: interpret negative values as last x values id = 0 @@ -510,7 +510,7 @@ func (st *Store) GetUpdatesAfter(id, limit int) (updates []*UpdateFull, err erro return err } updates = append(updates, duf) - if len(updates) >= limit { + if int64(len(updates)) >= limit { return nil } } @@ -519,7 +519,7 @@ func (st *Store) GetUpdatesAfter(id, limit int) (updates []*UpdateFull, err erro return } -func (st *Store) GetUpdate(id int) (update *UpdateFull, err error) { +func (st *Store) GetUpdate(id int64) (update *UpdateFull, err error) { err = st.db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(updatesBn)) @@ -550,15 +550,15 @@ func (st *Store) GetHubUUID() string { return st.hubUUID } -func (st *Store) GetLastUpdateID() (updateID int, err error) { +func (st *Store) GetLastUpdateID() (updateID int64, err error) { err = st.db.View(func(tx *bolt.Tx) error { - updateID = int(tx.Bucket([]byte(updatesBn)).Sequence()) + updateID = int64(tx.Bucket([]byte(updatesBn)).Sequence()) return nil }) return } -func (st *Store) GetLastUpdateIDForUUID(uuid string) (updateID int, err error) { +func (st *Store) GetLastUpdateIDForUUID(uuid string) (updateID int64, err error) { if uuid == st.hubUUID { return st.GetLastUpdateID() } diff --git a/src/hub/src/spreadspace.org/sfive/s5store_test.go b/src/hub/src/spreadspace.org/sfive/s5store_test.go index 3630ac0..2e69c89 100644 --- a/src/hub/src/spreadspace.org/sfive/s5store_test.go +++ b/src/hub/src/spreadspace.org/sfive/s5store_test.go @@ -342,7 +342,7 @@ func TestAppendAndFetch(t *testing.T) { t.Fatalf("failed to append update: %v", err) } - for i := 0; i < 2; i = i + 1 { + for i := int64(0); i < 2; i = i + 1 { out, err = store.GetUpdate(i + 2) if err != nil { t.Fatalf("failed to fetch update: %v", err) @@ -415,7 +415,7 @@ func TestGetUpdatesAfter(t *testing.T) { upd.Data.Clients = clientsData expected := []*UpdateFull{} - for i := 0; i < 3; i = i + 1 { + for i := int64(0); i < 3; i = i + 1 { in := &UpdateFull{Header{0, "", -1, "", -1}, sourceData, upd} if err = store.Append(in); err != nil { t.Fatalf("unexpected error: %v", err) @@ -511,7 +511,7 @@ func TestForwardedUpdates(t *testing.T) { if err := store.AppendMany(data); err != nil { t.Fatalf("unexpected error: %v", err) } - myLastID := 10 + myLastID := int64(10) forwardedHub := "05defdfa-e7d1-4ca8-8b5c-02abb0088d29" // check if there are no updates for this hub in store @@ -541,7 +541,7 @@ func TestForwardedUpdates(t *testing.T) { upd.Data.Clients = clientsData expected := []*UpdateFull{} - for i := 0; i < 3; i = i + 1 { + for i := int64(0); i < 3; i = i + 1 { in := &UpdateFull{Header{0, "", -1, "", -1}, sourceData, upd} in.SourceHubUUID = forwardedHub in.SourceHubUpdateID = 3 - i // out of order @@ -597,7 +597,7 @@ func TestForwardedUpdates(t *testing.T) { } } -func checkForwardedUpdates2(t *testing.T, src1Store, src2Store, fwdStore, finalStore *Store, fwdSrc1ID, fwdSrc2ID, finalSrc1ID, finalSrc2ID, finalFwdID int) { +func checkForwardedUpdates2(t *testing.T, src1Store, src2Store, fwdStore, finalStore *Store, fwdSrc1ID, fwdSrc2ID, finalSrc1ID, finalSrc2ID, finalFwdID int64) { lastID, err := fwdStore.GetLastUpdateIDForUUID(src1Store.GetHubUUID()) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -844,7 +844,7 @@ func BenchmarkGetUpdatesAfter(b *testing.B) { b.ResetTimer() - latestID := -1 + latestID := int64(-1) for { updates, err := store.GetUpdatesAfter(latestID, -1) if err != nil { diff --git a/src/hub/src/spreadspace.org/sfive/s5typesApi.go b/src/hub/src/spreadspace.org/sfive/s5typesApi.go index 5afb13e..f393f15 100644 --- a/src/hub/src/spreadspace.org/sfive/s5typesApi.go +++ b/src/hub/src/spreadspace.org/sfive/s5typesApi.go @@ -83,9 +83,9 @@ type Update struct { type Header struct { Version uint `json:"version,omitempty"` // omitempty is needed for data only messages and for REST API SourceHubUUID string `json:"SourceHubUuid,omitempty"` - SourceHubUpdateID int `json:"SourceHubUpdateId,omitempty"` + SourceHubUpdateID int64 `json:"SourceHubUpdateId,omitempty"` ForwardHubUUID string `json:"ForwardHubUuid,omitempty"` - ForwardHubUpdateID int `json:"ForwardHubUpdateId,omitempty"` + ForwardHubUpdateID int64 `json:"ForwardHubUpdateId,omitempty"` } type UpdateFull struct { @@ -156,10 +156,10 @@ type WebUpdatesGetResponse struct { } type WebUpdatesPostResponse struct { - NumUpdates int `json:"num-updates"` + NumUpdates int64 `json:"num-updates"` } type WebLastUpdateIDResponse struct { HubUUID string `json:"hub-uuid"` - LastUpdateID int `json:"lastupdate"` + LastUpdateID int64 `json:"lastupdate"` } diff --git a/src/hub/src/spreadspace.org/sfive/s5typesStore.go b/src/hub/src/spreadspace.org/sfive/s5typesStore.go index dbf4c4f..9996018 100644 --- a/src/hub/src/spreadspace.org/sfive/s5typesStore.go +++ b/src/hub/src/spreadspace.org/sfive/s5typesStore.go @@ -147,15 +147,15 @@ func (c *Client) CopyFromClientDB(cdb *clientDB) { } type clientDataDB struct { - ClientID int `json:"ci"` - UserAgentID int `json:"ua"` - BytesSent uint `json:"bs"` + ClientID int64 `json:"ci"` + UserAgentID int64 `json:"ua"` + BytesSent uint `json:"bs"` } type updateDB struct { - SourceHubID int `json:"h,omitempty"` - SourceHubUpdateID int `json:"hi,omitempty"` - SourceID int `json:"si"` + SourceHubID int64 `json:"h,omitempty"` + SourceHubUpdateID int64 `json:"hi,omitempty"` + SourceID int64 `json:"si"` StartTime int64 `json:"st"` // unix timestamp in milliseconds Duration int64 `json:"du"` // duration in milliseconds ClientCount uint `json:"cc,omitempty"` @@ -176,7 +176,7 @@ func NewUpdateDB(uf *UpdateFull) *updateDB { } } -func (uf *UpdateFull) CopyFromUpdateDB(udb *updateDB, srcHubUUID, hubUUID string, id int) { +func (uf *UpdateFull) CopyFromUpdateDB(udb *updateDB, srcHubUUID, hubUUID string, id int64) { if srcHubUUID == "" { uf.SourceHubUUID = hubUUID uf.SourceHubUpdateID = id @@ -194,12 +194,12 @@ func (uf *UpdateFull) CopyFromUpdateDB(udb *updateDB, srcHubUUID, hubUUID string uf.Data.BytesSent = udb.BytesSent } -func itob(v int) []byte { +func itob(v int64) []byte { b := make([]byte, 8) binary.BigEndian.PutUint64(b, uint64(v)) return b } -func btoi(b []byte) int { - return int(binary.BigEndian.Uint64(b)) +func btoi(b []byte) int64 { + return int64(binary.BigEndian.Uint64(b)) } |