revamp test scripts, simpified tag insertion

14 files changed, 76 insertions, 59 deletions

diff --git a/src/hub/.gitignore b/src/hub/.gitignore
index 16dc168..7be1b96 100644
--- a/src/hub/.gitignore
+++ b/src/hub/.gitignore
@@ -5,3 +5,4 @@
 /pkg
 *.a
 *.o
+/test
diff --git a/src/hub/dump-test-db b/src/hub/dump-test-db
deleted file mode 100755
index 9a7aae7..0000000
--- a/src/hub/dump-test-db
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-echo .dump | sqlite3 /var/lib/sfive/db.sqlite
-
diff --git a/src/hub/src/spreadspace.org/sfive/s5store.go b/src/hub/src/spreadspace.org/sfive/s5store.go
index a3bd8ce..6d6a9ad 100644
--- a/src/hub/src/spreadspace.org/sfive/s5store.go
+++ b/src/hub/src/spreadspace.org/sfive/s5store.go
@@ -7,6 +7,10 @@ import (
 	"fmt"
 	"time"
 
+	// needed for gorp tracing
+	// "log"
+	// "os"
+
 	"github.com/boltdb/bolt"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/pborman/uuid"
@@ -204,15 +208,11 @@ func (s sqliteStore) findTag(name string) (tag *tagDb, err error) {
 func (s sqliteStore) insertNewTags(tags []tagDb) (err error) {
 	for i := range tags {
 		var t *tagDb
-		t, err = s.findTag(tags[i].Name)
-		if err != nil {
-			_, err = s.db.Exec("insert into "+tagsTn+" VALUES (NULL, ?)", tags[i].Name)
-		}
-		t, err = s.findTag(tags[i].Name)
-
-		if err == nil {
+		if t, err = s.findTag(tags[i].Name); err == nil {
 			tags[i] = *t
-		} else {
+			continue
+		}
+		if err = s.db.Insert(&(tags[i])); err != nil {
 			break
 		}
 	}
@@ -239,14 +239,11 @@ func (s sqliteStore) findSource(src sourceDb) (res *sourceDb, err error) {
 
 func (s sqliteStore) insertNewSource(src *sourceDb) (err error) {
 	var t *sourceDb
-	t, err = s.findSource(*src)
-	if err == nil {
+	if t, err = s.findSource(*src); err == nil {
 		*src = *t
-	} else {
-		err = s.db.Insert(src)
+		return
 	}
-
-	return
+	return s.db.Insert(src)
 }
 
 func (s sqliteStore) insertSourceTagLinks(src sourceDb, tags []tagDb) (err error) {
@@ -299,52 +296,32 @@ func (s sqliteStore) insertDataUpdateClientEntries(cd []ClientData, du dataUpdat
 }
 
 func (s sqliteStore) appendItem(du dataUpdateDb, cd []ClientData, src sourceDb, tags []tagDb) (err error) {
-	err = s.insertNewTags(tags)
-	if err != nil {
+	if err = s.insertNewTags(tags); err != nil {
 		return
 	}
 
-	err = s.insertNewSource(&src)
-	if err != nil {
+	if err = s.insertNewSource(&src); err != nil {
 		//fmt.Printf("src\n")
 		return
 	}
 
-	err = s.insertSourceTagLinks(src, tags)
-	if err != nil {
+	if err = s.insertSourceTagLinks(src, tags); err != nil {
 		return
 	}
 
-	err = s.insertDataUpdateEntry(src, &du)
-	if err != nil {
+	if err = s.insertDataUpdateEntry(src, &du); err != nil {
 		return
 	}
 
-	err = s.insertDataUpdateClientEntries(cd, du)
-	if err != nil {
+	if err = s.insertDataUpdateClientEntries(cd, du); err != nil {
 		return
 	}
 
 	return
 }
 
-// this function is the biggest pile of copy/pasted crap while sick that is still compilable.
 func (s sqliteStore) Append(update StatisticsData) (err error) {
-	var tx *gorp.Transaction
-	tx, err = s.db.Begin()
-	if err != nil {
-		return
-	}
-
-	du, cd, src, tags := updateFromStatisticsData(update)
-	//s5l.Printf("blah: %v", du)
-	err = s.appendItem(du, cd, src, tags)
-	if err != nil {
-		tx.Rollback()
-		return
-	}
-
-	return tx.Commit()
+	return s.AppendMany([]StatisticsData{update})
 }
 
 func (s sqliteStore) AppendMany(updates []StatisticsData) (err error) {
diff --git a/src/hub/src/spreadspace.org/sfive/s5store_test.go b/src/hub/src/spreadspace.org/sfive/s5store_test.go
index 74721c4..92ad149 100644
--- a/src/hub/src/spreadspace.org/sfive/s5store_test.go
+++ b/src/hub/src/spreadspace.org/sfive/s5store_test.go
@@ -102,7 +102,7 @@ func TestGetUpdatesAfter(t *testing.T) {
 
 	err = store.Append(dat)
 	if err != nil {
-		t.Errorf("Failed to retrieve: %v", err)
+		t.Errorf("Failed to append: %v", err)
 		return
 	}
 
diff --git a/src/hub/test-bolt b/src/hub/test-bolt
new file mode 100755
index 0000000..7ac3dd2
--- /dev/null
+++ b/src/hub/test-bolt
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+TEST_D="./test"
+
+BIN="$(go env GOPATH)/bin/bolt"
+if [ ! -x "$BIN" ]; then
+  echo "bolt not found. Please run:"
+  echo ""
+  echo "  go get -u github.com/boltdb/bolt/..."
+  echo ""
+  exit 1
+fi
+
+exec "$(go env GOPATH)/bin/bolt" $@ "$TEST_D/db.bolt"
diff --git a/src/hub/test-bolter b/src/hub/test-bolter
new file mode 100755
index 0000000..dc2e98d
--- /dev/null
+++ b/src/hub/test-bolter
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+TEST_D="./test"
+
+BIN="$(go env GOPATH)/bin/bolter"
+if [ ! -x "$BIN" ]; then
+  echo "bolter not found. Please run:"
+  echo ""
+  echo "  go get -u github.com/hasit/bolter"
+  echo ""
+  exit 1
+fi
+
+exec $BIN --file "$TEST_D/db.bolt" $@
diff --git a/src/hub/test-client b/src/hub/test-client
index 8a24c7d..d5756e7 100755
--- a/src/hub/test-client
+++ b/src/hub/test-client
@@ -1,11 +1,14 @@
 #!/bin/sh
+
+TEST_D="./test"
+
 echo pipe: import sample.json
 echo ------------------------
-socat file:../../dat/sample.json,rdonly unix-client:/run/sfive/pipe
+socat file:../../dat/sample.json,rdonly "unix-client:$TEST_D/pipe"
 
 echo pipe-gram: import sample-gram.json
 echo ----------------------------------
-while read x; do echo "$x" | socat stdio unix-sendto:/run/sfive/pipegram; done < ../../dat/sample-gram.json
+while read x; do echo "$x" | socat stdio "unix-sendto:$TEST_D/pipegram"; done < ../../dat/sample-gram.json
 
 echo show query result
 echo -----------------
@@ -20,4 +23,3 @@ echo ----------
 curl -i 'http://localhost:8000/stats'
 
 echo '\n\ndone'
-
diff --git a/src/hub/test-collector b/src/hub/test-collector
deleted file mode 100755
index e6ca367..0000000
--- a/src/hub/test-collector
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/sh
-
-./bin/sfive-hub -db "file:memdb1?mode=memory&cache=shared" -start-pipe-server=false -start-pipegram-server=false -start-web-server -viz-dir "$(pwd)/../viz" -bind=":8000"
-
diff --git a/src/hub/test-fwd b/src/hub/test-fwd
new file mode 100755
index 0000000..eff7605
--- /dev/null
+++ b/src/hub/test-fwd
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+TEST_D="./test"
+
+exec ./bin/sfive-hub -db "$TEST_D" -start-pipe-server=false -start-pipegram-server=false -start-web-server=false -forward-url="http://localhost:8000"
diff --git a/src/hub/test-fwd-es b/src/hub/test-fwd-es
index 3f3eb12..dc7979f 100755
--- a/src/hub/test-fwd-es
+++ b/src/hub/test-fwd-es
@@ -1,5 +1,5 @@
 #!/bin/sh
 
-rm -f /run/sfive/pipe /run/sfive/pipegram
-./bin/sfive-hub -start-pipe-server=false -start-pipegram-server=false -start-web-server=false -db db.sqlite -forward-es-url="http://stream.elevate.at:9200/e14"
+TEST_D="./test"
 
+exec ./bin/sfive-hub -db "$TEST_D" -start-pipe-server=false -start-pipegram-server=false -start-web-server=false -forward-es-url="http://stream.elevate.at:9200/e14"
diff --git a/src/hub/test-fwd-piwik b/src/hub/test-fwd-piwik
index 1d45219..b6ac640 100755
--- a/src/hub/test-fwd-piwik
+++ b/src/hub/test-fwd-piwik
@@ -1,4 +1,5 @@
 #!/bin/sh
 
-rm -f /run/sfive/pipe /run/sfive/pipegram
-./bin/sfive-hub -start-pipe-server=false -start-pipegram-server=false -start-web-server=false -db db.sqlite -forward-piwik-url="http://localhost/piwik.php" -piwik-token "asdfjlkasjdflk" -piwik-site-id 4 -piwik-site-url "https://stream.elevate.at"
+TEST_D="./test"
+
+exec ./bin/sfive-hub -db "$TEST_D" -start-pipe-server=false -start-pipegram-server=false -start-web-server=false -forward-piwik-url="http://localhost/piwik.php" -piwik-token "asdfjlkasjdflk" -piwik-site-id 4 -piwik-site-url "https://stream.elevate.at"
diff --git a/src/hub/test-import b/src/hub/test-import
index d1a5044..a6c2942 100755
--- a/src/hub/test-import
+++ b/src/hub/test-import
@@ -1,10 +1,12 @@
 #!/bin/sh
+
+TEST_D="./test"
+
 echo pipe: import sample.json
 echo ------------------------
 
 [ -f ../../dat/sample-access.json ] || zcat ../../dat/sample-accesslog.json.gz > ../../dat/sample-accesslog.json
 
-socat file:../../dat/sample-accesslog.json,rdonly unix-client:/run/sfive/pipe
+socat file:../../dat/sample-accesslog.json,rdonly "unix-client:$TEST_D/pipe"
 
 echo '\n\ndone'
-
diff --git a/src/hub/test-sqlite b/src/hub/test-sqlite
new file mode 100755
index 0000000..1c2dcb1
--- /dev/null
+++ b/src/hub/test-sqlite
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+TEST_D="./test"
+
+exec sqlite3 "$TEST_D/db.sqlite"
diff --git a/src/hub/test-srv b/src/hub/test-srv
index 254549a..37d5897 100755
--- a/src/hub/test-srv
+++ b/src/hub/test-srv
@@ -1,4 +1,7 @@
 #!/bin/sh
 
-rm -f /run/sfive/pipe /run/sfive/pipegram
-./bin/sfive-hub -db /var/lib/sfive/db.sqlite -start-pipe-server -pipe /var/run/sfive/pipe -start-pipegram-server -pipegram /var/run/sfive/pipegram -start-web-server -viz-dir "$(pwd)/../viz" -bind=":8001"
+TEST_D="./test"
+
+mkdir -p "$TEST_D"
+rm -f "$TEST_D/pipe" "$TEST_D/pipegram"
+exec ./bin/sfive-hub -db "$TEST_D" -start-pipe-server -pipe "$TEST_D/pipe" -start-pipegram-server -pipegram "$TEST_D/pipegram" -start-web-server -viz-dir "$(pwd)/../viz" -bind=":8000"