fix, refactor s5store

- only insert new tags
- only insert new sources
- split .Append into smaller parts

1 files changed, 96 insertions, 24 deletions

diff --git a/src/hub/src/spreadspace.org/sfive/s5store.go b/src/hub/src/spreadspace.org/sfive/s5store.go
index 9d3d57e..1480f48 100644
--- a/src/hub/src/spreadspace.org/sfive/s5store.go
+++ b/src/hub/src/spreadspace.org/sfive/s5store.go
@@ -3,7 +3,6 @@ package sfive
 import (
 	"database/sql"
 	"log"
-	"os"
 	"time"
 
 	_ "github.com/mattn/go-sqlite3"
@@ -120,7 +119,7 @@ func initDb() *gorp.DbMap {
 
 	dbmap.AddTableWithName(tagDb{}, TagsTn).SetKeys(true, "Id").ColMap("Name").SetUnique(true)
 	dbmap.AddTableWithName(sourceTagsDb{}, SourceTagsTn).SetKeys(false, "TagId", "SourceId")
-	dbmap.AddTableWithName(sourceDb{}, SourcesTn).SetKeys(true, "Id")
+	dbmap.AddTableWithName(sourceDb{}, SourcesTn).SetKeys(true, "Id").SetUniqueTogether("ContentId", "Format", "Quality", "Hostname")
 	dbmap.AddTableWithName(clientDataDb{}, ClientDataUpdatesTn).SetKeys(true, "Id")
 	dbmap.AddTableWithName(dataUpdateDb{}, DataUpdatesTn).SetKeys(true, "Id")
 
@@ -133,7 +132,7 @@ func initDb() *gorp.DbMap {
 
 func checkErr(err error, msg string) {
 	if err != nil {
-		log.Fatalln(msg, err)
+		log.Panicln(msg, err)
 	}
 }
 
@@ -147,10 +146,6 @@ type StatsFilter struct {
 	tagsAny   []string
 }
 
-type Closer interface {
-	Close()
-}
-
 type StatsContainer interface {
 	Append(update StatisticsData) error
 	CountUpdateEntries() (int64, error)
@@ -158,36 +153,69 @@ type StatsContainer interface {
 	ClientCount(filter *StatsFilter) uint
 	AverageBps(filter *StatsFilter) (uint, error)
 	Locations(filter *StatsFilter) map[string]int
+	Close()
 }
 
 type sqliteStore struct {
 	db *gorp.DbMap
 }
 
-func (s sqliteStore) Append(update StatisticsData) (err error) {
-	du, cd, src, tags := updateFromStatisticsData(update)
-
-	s.db.TraceOn("", log.New(os.Stdout, "gorptest: ", log.Lmicroseconds))
-	tx, err := s.db.Begin()
-	if err != nil {
-		//fmt.Printf("tx\n")
-		return
+func (s sqliteStore) findTag(name string) (tag *tagDb, err error) {
+	t := tagDb{}
+	err = s.db.SelectOne(&t, "select * from "+TagsTn+" where Name = ?", name)
+	if err == nil {
+		tag = &t
 	}
+	return
+}
 
+func (s sqliteStore) insertNewTags(tags []tagDb) (err error) {
 	for i := range tags {
-		err = s.db.Insert(&tags[i])
+		t, err := s.findTag(tags[i].Name)
 		if err != nil {
-			//fmt.Printf("tags\n")
-			return
+			_, err = s.db.Exec("insert into "+TagsTn+" VALUES (NULL, ?)", tags[i].Name)
+		}
+		t, err = s.findTag(tags[i].Name)
+
+		if err == nil {
+			tags[i] = *t
+		} else {
+			break
 		}
 	}
 
-	err = s.db.Insert(&src)
-	if err != nil {
-		//fmt.Printf("src\n")
-		return
+	return
+}
+
+func (s sqliteStore) findSource(src sourceDb) (res *sourceDb, err error) {
+	t := sourceDb{}
+	err = s.db.SelectOne(
+		&t,
+		"select Id from "+SourcesTn+" where ContentId = ? and Format = ? and Quality = ? and Hostname = ?",
+		src.ContentId,
+		src.Format,
+		src.Quality,
+		src.Hostname)
+
+	if err == nil {
+		res = &t
 	}
 
+	return
+}
+
+func (s sqliteStore) insertNewSource(src *sourceDb) (err error) {
+	t, err := s.findSource(*src)
+	if err == nil {
+		*src = *t
+	} else {
+		err = s.db.Insert(src)
+	}
+
+	return
+}
+
+func (s sqliteStore) insertSourceTagLinks(src sourceDb, tags []tagDb) (err error) {
 	st := make([]sourceTagsDb, len(tags))
 	for i := range tags {
 		st[i].TagId = tags[i].Id
@@ -200,22 +228,66 @@ func (s sqliteStore) Append(update StatisticsData) (err error) {
 			return
 		}
 	}
+	return
+}
 
+func (s sqliteStore) insertDataUpdateEntry(src sourceDb, du *dataUpdateDb) (err error) {
 	du.SourceId = src.Id
-	err = s.db.Insert(&du)
+	err = s.db.Insert(du)
 	if err != nil {
 		//fmt.Printf("du\n")
 		return
 	}
+	return
+}
 
+func (s sqliteStore) insertDataUpdateClientEntries(cd []clientDataDb, du dataUpdateDb) (err error) {
 	for i := range cd {
 		cd[i].DataUpdatesId = du.Id
 		err = s.db.Insert(&cd)
 		if err != nil {
 			return
 		}
+	}
+	return
+}
+
+// this function is the biggest pile of copy/pasted crap while sick that is still compilable.
+func (s sqliteStore) Append(update StatisticsData) (err error) {
+	du, cd, src, tags := updateFromStatisticsData(update)
+
+	// s.db.TraceOn("", log.New(os.Stdout, "gorptest: ", log.Lmicroseconds))
+	tx, err := s.db.Begin()
+	if err != nil {
+		return
+	}
+
+	err = s.insertNewTags(tags)
+	if err != nil {
+		return
+	}
 
+	err = s.insertNewSource(&src)
+	if err != nil {
+		//fmt.Printf("src\n")
+		return
 	}
+
+	err = s.insertSourceTagLinks(src, tags)
+	if err != nil {
+		return
+	}
+
+	err = s.insertDataUpdateEntry(src, &du)
+	if err != nil {
+		return
+	}
+
+	err = s.insertDataUpdateClientEntries(cd, du)
+	if err != nil {
+		return
+	}
+
 	return tx.Commit()
 }
 
@@ -289,6 +361,6 @@ func EatDataAndClose(sc StatsContainer) {
 	s.Close()
 }
 
-func (s *sqliteStore) Close() {
+func (s sqliteStore) Close() {
 	s.db.Db.Close()
 }