Fixed several issues, contineued web work

Fixed lots of issues with thread saftey and had to rethink some stuff

Fixed order issues with timekeys
Starting to flesh out the web REST API

cycle.go

@@ -5,12 +5,9 @@
 package main
 
 import (
-	"crypto/sha1"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -20,83 +17,6 @@ import (
 	"git.townsourced.com/ironsmith/datastore"
 )
 
-func (p *Project) errHandled(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	vlog("Error in project %s: %s\n", p.id(), err)
-
-	if p.ds == nil {
-		log.Printf("Error in project %s: %s\n", p.id(), err)
-		return true
-	}
-	defer func() {
-		//clean up version folder if it exists
-
-		if p.version != "" {
-			err = os.RemoveAll(p.workingDir())
-			if err != nil {
-				log.Printf("Error deleting the version directory project %s version %s: %s\n",
-					p.id(), p.version, err)
-			}
-
-		}
-	}()
-
-	lerr := p.ds.AddLog(p.version, p.stage, err.Error())
-	if lerr != nil {
-		log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s",
-			p.id(), err, lerr)
-	}
-
-	return true
-}
-
-func (p *Project) id() string {
-	if p.filename == "" {
-		panic("invalid project filename")
-	}
-	return strings.TrimSuffix(p.filename, filepath.Ext(p.filename))
-}
-
-func (p *Project) dir() string {
-	return filepath.Join(dataDir, p.id())
-}
-
-func (p *Project) workingDir() string {
-	if p.hash == "" {
-		panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
-	}
-
-	//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
-	// timestamp on instead would work just fine, but I like having the working dir tied directly to the
-	// version returned by project script
-
-	return filepath.Join(p.dir(), p.hash)
-}
-
-// prepData makes sure the project's data folder and data store is created
-/*
-	folder structure
-	projectDataFolder/<project-name>/<project-version>
-
-*/
-func (p *Project) prepData() error {
-	err := os.MkdirAll(p.dir(), 0777)
-	if err != nil {
-		return err
-	}
-
-	p.ds, err = datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith"))
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 /*
 Project life cycle:
 (Load Project file) -> (Fetch) -> (Build) -> (Test) -> (Release) - > (Sleep for polling period) ->
@@ -107,20 +27,19 @@ Project life cycle:
 // load is the beginning of the cycle.  Loads / reloads the project file to make sure that the scripts are up-to-date
 // call's fetch and triggers the next poll if one exists
 func (p *Project) load() {
-	p.version = ""
+	p.setStage(stageLoad)
-	p.hash = ""
+	p.setVersion("")
-
-	vlog("Entering %s stage for Project: %s\n", stageLoad, p.id())
 
 	if p.filename == "" {
 		p.errHandled(errors.New("Invalid project file name"))
 		return
 	}
 
-	if !projects.exists(p.filename) {
+	if _, ok := projects.get(p.id()); !ok {
 		// project has been deleted
 		// don't continue polling
 		// move project data to deleted folder with a timestamp
+		p.close()
 		p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir,
 			strconv.FormatInt(time.Now().Unix(), 10), p.id())))
 		return
@@ -131,30 +50,19 @@ func (p *Project) load() {
 		return
 	}
 
-	if p.errHandled(json.Unmarshal(data, p)) {
+	new := &Project{}
+	if p.errHandled(json.Unmarshal(data, new)) {
 		return
 	}
 
-	p.stage = stageLoad
+	p.setData(new)
 
-	if p.errHandled(p.prepData()) {
+	if p.errHandled(os.MkdirAll(p.dir(), 0777)) {
 		return
 	}
 
-	if p.PollInterval != "" {
-		p.poll, err = time.ParseDuration(p.PollInterval)
-		if p.errHandled(err) {
-			p.poll = 0
-		}
-	}
-
 	p.fetch()
 
-	if p.errHandled(p.ds.Close()) {
-		return
-	}
-	p.ds = nil
-
 	//full cycle completed
 
 	if p.poll > 0 {
@@ -167,9 +75,8 @@ func (p *Project) load() {
 // then it runs the version script in the temp directory to see if there is a newer version of the
 // fetched code, if there is then the temp dir is renamed to the version name
 func (p *Project) fetch() {
-	p.stage = stageFetch
+	p.setStage(stageFetch)
 
-	vlog("Entering %s stage for Project: %s\n", p.stage, p.id())
 	tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10))
 
 	if p.errHandled(os.MkdirAll(tempDir, 0777)) {
@@ -189,9 +96,9 @@ func (p *Project) fetch() {
 		return
 	}
 
-	p.version = strings.TrimSpace(string(version))
+	p.setVersion(strings.TrimSpace(string(version)))
 
-	lVer, err := p.ds.LatestVersion()
+	lVer, err := p.ds.LastVersion("")
 	if err != datastore.ErrNotFound && p.errHandled(err) {
 		return
 	}
@@ -204,8 +111,6 @@ func (p *Project) fetch() {
 		return
 	}
 
-	p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(p.version)))
-
 	//remove any existing data that matches version hash
 	if p.errHandled(os.RemoveAll(p.workingDir())) {
 		return
@@ -225,14 +130,12 @@ func (p *Project) fetch() {
 
 	// continue to build
 	p.build()
-
 }
 
 // build  runs the build scripts to build the project which should result in the a single file
 // configured in the ReleaseFile section of the project file
 func (p *Project) build() {
-	p.stage = stageBuild
+	p.setStage(stageBuild)
-	vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
 
 	output, err := runCmd(p.Build, p.workingDir())
 
@@ -250,9 +153,7 @@ func (p *Project) build() {
 
 // test runs the test scripts
 func (p *Project) test() {
-	p.stage = stageTest
+	p.setStage(stageTest)
-	vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
-
 	output, err := runCmd(p.Test, p.workingDir())
 
 	if p.errHandled(err) {
@@ -265,13 +166,11 @@ func (p *Project) test() {
 
 	//  Tests passed, onto release
 	p.release()
-
 }
 
 // release runs the release scripts and builds the release file
 func (p *Project) release() {
-	p.stage = stageRelease
+	p.setStage(stageRelease)
-	vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
 
 	output, err := runCmd(p.Release, p.workingDir())
 

datastore/key.go

@@ -29,10 +29,6 @@ func NewTimeKey() TimeKey {
 	nsec := t.Nanosecond()
 
 	return TimeKey{
-		rBits[0], //random
-		rBits[1],
-		rBits[2],
-		rBits[3],
 		byte(sec >> 56), // seconds
 		byte(sec >> 48),
 		byte(sec >> 40),
@@ -45,12 +41,16 @@ func NewTimeKey() TimeKey {
 		byte(nsec >> 16),
 		byte(nsec >> 8),
 		byte(nsec),
+		rBits[0], //random
+		rBits[1],
+		rBits[2],
+		rBits[3],
 	}
 }
 
 // Time returns the time portion of a timekey
 func (k TimeKey) Time() time.Time {
-	buf := k[4:]
+	buf := k[:]
 
 	sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 |
 		int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56

datastore/log.go

@@ -11,11 +11,12 @@ import (
 	"github.com/boltdb/bolt"
 )
 
-type log struct {
+// Log is a version log entry for a project
-	When    time.Time `json:"when"`
+type Log struct {
-	Version string    `json:"version"`
+	When    time.Time `json:"when,omitempty"`
-	Stage   string    `json:"stage"`
+	Version string    `json:"version,omitempty"`
-	Log     string    `json:"log"`
+	Stage   string    `json:"stage,omitempty"`
+	Log     string    `json:"log,omitempty"`
 }
 
 const bucketLog = "log"
@@ -24,7 +25,7 @@ const bucketLog = "log"
 func (ds *Store) AddLog(version, stage, entry string) error {
 	key := NewTimeKey()
 
-	data := &log{
+	data := &Log{
 		When:    key.Time(),
 		Version: version,
 		Stage:   stage,
@@ -34,25 +35,28 @@ func (ds *Store) AddLog(version, stage, entry string) error {
 	return ds.put(bucketLog, key, data)
 }
 
-// LatestVersion returns the latest version (successful or otherwise) for the current project
+// LastVersion returns the last version in the log for the given stage.  If stage is blank,
-func (ds *Store) LatestVersion() (string, error) {
+// then it returns the last of any stage
+func (ds *Store) LastVersion(stage string) (string, error) {
 	version := ""
 
 	err := ds.bolt.View(func(tx *bolt.Tx) error {
 		c := tx.Bucket([]byte(bucketLog)).Cursor()
 
-		for k, v := c.First(); k != nil; k, v = c.Next() {
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			l := &log{}
+			l := &Log{}
 			err := json.Unmarshal(v, l)
 			if err != nil {
 				return err
 			}
 
 			if l.Version != "" {
+				if stage == "" || l.Stage == stage {
 					version = l.Version
 					return nil
 				}
 			}
+		}
 
 		return ErrNotFound
 	})
@@ -63,3 +67,38 @@ func (ds *Store) LatestVersion() (string, error) {
 
 	return version, nil
 }
+
+// Versions lists the versions in a given project, including the last stage that version got to
+func (ds *Store) Versions() ([]*Log, error) {
+	var vers []*Log
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		var current = ""
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			l := &Log{}
+			err := json.Unmarshal(v, l)
+			if err != nil {
+				return err
+			}
+
+			// capture the newest entry for each version
+			if l.Version != current {
+				l.Log = "" // only care about date, ver and stage
+				vers = append(vers, l)
+				current = l.Version
+			}
+
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return vers, nil
+}

main.go

@@ -37,7 +37,7 @@ func init() {
 	go func() {
 		for sig := range c {
 			if sig == os.Interrupt {
-				projects.closeAll()
+				projects.stopAll()
 				os.Exit(0)
 			}
 		}
@@ -91,7 +91,6 @@ func main() {
 	}
 
 	//start web server
-
 	err = startServer()
 	if err != nil {
 		log.Fatalf("Error Starting web server: %s", err)

project.go

@@ -5,10 +5,13 @@
 package main
 
 import (
+	"crypto/sha1"
 	"encoding/json"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
@@ -51,8 +54,8 @@ type Project struct {
 	Version string `json:"version"` //Script to generate the version num of the current build, should be indempotent
 
 	ReleaseFile   string `json:"releaseFile"`
-	PollInterval  string `json:"pollInterval"`  // if not poll interval is specified, this project is trigger only
+	PollInterval  string `json:"pollInterval,omitempty"`  // if not poll interval is specified, this project is trigger only
-	TriggerSecret string `json:"triggerSecret"` //secret to be included with a trigger call
+	TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call
 
 	filename string
 	poll     time.Duration
@@ -60,6 +63,195 @@ type Project struct {
 	stage    string
 	version  string
 	hash     string
+
+	sync.RWMutex
+}
+
+func (p *Project) errHandled(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	vlog("Error in project %s: %s\n", p.id(), err)
+
+	if p.ds == nil {
+		log.Printf("Error in project %s: %s\n", p.id(), err)
+		return true
+	}
+	defer func() {
+		//clean up version folder if it exists
+		if p.version != "" {
+			err = os.RemoveAll(p.workingDir())
+			if err != nil {
+				log.Printf("Error deleting the version directory project %s version %s: %s\n",
+					p.id(), p.version, err)
+			}
+
+		}
+	}()
+
+	lerr := p.ds.AddLog(p.version, p.stage, err.Error())
+	if lerr != nil {
+		log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s",
+			p.id(), err, lerr)
+	}
+
+	return true
+}
+
+func projectID(filename string) string {
+	return strings.TrimSuffix(filename, filepath.Ext(filename))
+}
+
+func (p *Project) id() string {
+	if p.filename == "" {
+		panic("invalid project filename")
+	}
+	return projectID(p.filename)
+}
+
+func (p *Project) dir() string {
+	return filepath.Join(dataDir, p.id())
+}
+
+func (p *Project) workingDir() string {
+	if p.hash == "" {
+		panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
+	}
+
+	//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
+	// timestamp on instead would work just fine, but I like having the working dir tied directly to the
+	// version returned by project script
+
+	return filepath.Join(p.dir(), p.hash)
+}
+
+// prepData makes sure the project's data folder and data store is created
+/*
+	folder structure
+	projectDataFolder/<project-name>/<project-version>
+
+*/
+func (p *Project) open() error {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.ds != nil {
+		return nil
+	}
+
+	ds, err := datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith"))
+	if err != nil {
+		return err
+	}
+
+	p.ds = ds
+
+	return nil
+}
+
+func (p *Project) setVersion(version string) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.version = version
+	if version == "" {
+		p.hash = ""
+		return
+	}
+
+	p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(version)))
+}
+
+func (p *Project) setStage(stage string) {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.version != "" {
+		vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
+	} else {
+		vlog("Entering %s stage for Project: %s\n", p.stage, p.id())
+	}
+
+	p.stage = stage
+}
+
+type webProject struct {
+	ID             string `json:"id"`
+	Name           string `json:"name"`
+	ReleaseVersion string `json:"releaseVersion"` //last successfully released version
+	LastVersion    string `json:"lastVersion"`    //last version success or otherwise
+}
+
+func (p *Project) webData() (*webProject, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	last, err := p.ds.LastVersion("")
+	if err != nil {
+		return nil, err
+	}
+
+	release, err := p.ds.LastVersion(stageRelease)
+	if err != nil {
+		return nil, err
+	}
+
+	d := &webProject{
+		Name:           p.Name,
+		ID:             p.id(),
+		LastVersion:    last,
+		ReleaseVersion: release,
+	}
+
+	return d, nil
+}
+
+func (p *Project) versions() ([]*datastore.Log, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.Versions()
+}
+
+func (p *Project) setData(new *Project) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.Name = new.Name
+
+	p.Fetch = new.Fetch
+	p.Build = new.Build
+	p.Test = new.Test
+	p.Release = new.Release
+	p.Version = new.Version
+
+	p.ReleaseFile = new.ReleaseFile
+	p.PollInterval = new.PollInterval
+	p.TriggerSecret = new.TriggerSecret
+
+	if p.PollInterval != "" {
+		var err error
+		p.poll, err = time.ParseDuration(p.PollInterval)
+		if p.errHandled(err) {
+			p.poll = 0
+		}
+	}
+}
+
+func (p *Project) close() error {
+	p.Lock()
+	defer p.Unlock()
+	if p.ds == nil {
+		return nil
+	}
+	err := p.ds.Close()
+	if err != nil {
+		return err
+	}
+
+	p.ds = nil
+	return nil
 }
 
 const projectTemplateFilename = "template.project.json"
@@ -147,12 +339,12 @@ func (p *projectList) load() error {
 	return nil
 }
 
-func (p *projectList) exists(name string) bool {
+func (p *projectList) get(name string) (*Project, bool) {
 	p.RLock()
 	defer p.RUnlock()
 
-	_, ok := p.data[name]
+	prj, ok := p.data[name]
-	return ok
+	return prj, ok
 }
 
 func (p *projectList) add(name string) {
@@ -165,9 +357,14 @@ func (p *projectList) add(name string) {
 		Name:     name,
 		stage:    stageLoad,
 	}
-	p.data[name] = prj
+	p.data[projectID(name)] = prj
 
 	go func() {
+		err := prj.open()
+		if err != nil {
+			log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err)
+			return
+		}
 		prj.load()
 	}()
 }
@@ -180,7 +377,7 @@ func (p *projectList) removeMissing(names []string) {
 	for i := range p.data {
 		found := false
 		for k := range names {
-			if names[k] == i {
+			if projectID(names[k]) == i {
 				found = true
 			}
 		}
@@ -192,17 +389,36 @@ func (p *projectList) removeMissing(names []string) {
 	}
 }
 
-func (p *projectList) closeAll() {
+func (p *projectList) stopAll() {
 	p.RLock()
 	defer p.RUnlock()
 
 	for i := range p.data {
-		if p.data[i].ds != nil {
+		err := p.data[i].close()
-			_ = p.data[i].ds.Close()
+		if err != nil {
+			log.Printf("Error closing project datastore for Project: %s Error: %s\n", p.data[i].id(), err)
 		}
 	}
 }
 
+func (p *projectList) webList() ([]*webProject, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	list := make([]*webProject, 0, len(p.data))
+
+	for i := range p.data {
+		prj, err := p.data[i].webData()
+		if err != nil {
+			return nil, err
+		}
+
+		list = append(list, prj)
+	}
+
+	return list, nil
+}
+
 // startProjectLoader polls for new projects
 func startProjectLoader() {
 	dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
@@ -228,7 +444,7 @@ func startProjectLoader() {
 	for i := range files {
 		if !files[i].IsDir() && filepath.Ext(files[i].Name()) == ".json" {
 			names[i] = files[i].Name()
-			if !projects.exists(files[i].Name()) {
+			if _, ok := projects.get(projectID(files[i].Name())); !ok {
 				projects.add(files[i].Name())
 			}
 		}

server.go

@@ -82,7 +82,7 @@ Routes
 
 	/project/ - list all projects
 	/project/<project-id> - list all versions in a project, triggers new builds
-	/project/<project-id>/<version> - list combined output of all stages
+	/project/<project-id>/<version> - list combined output of all stages for a given version
 	/project/<project-id>/<version>/<stage. - list output of a given stage of a given version
 
 */

webProject.go

@@ -5,13 +5,12 @@
 package main
 
 import (
-	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
 )
 
-// /project/<project-id>/<version>/<stage>
+// /path/<project-id>/<version>/<stage>
 func splitPath(path string) (project, version, stage string) {
 	s := strings.Split(path, "/")
 	if len(s) < 3 {
@@ -33,8 +32,46 @@ func splitPath(path string) (project, version, stage string) {
 	return
 }
 
+// /project/*
 func projectGet(w http.ResponseWriter, r *http.Request) {
-	prj, ver, stg := splitPath(r.URL.Path)
+	prj, ver, _ := splitPath(r.URL.Path)
+
+	//values := r.URL.Query()
+
+	if prj == "" {
+		//get all projects
+		pList, err := projects.webList()
+		if errHandled(err, w) {
+			return
+		}
+
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   pList,
+		})
+
+		return
+	}
+
+	project, ok := projects.get(prj)
+	if !ok {
+		four04(w, r)
+		return
+	}
+
+	//project found
+
+	if ver == "" {
+		//list versions
+		vers, err := project.versions()
+		if errHandled(err, w) {
+			return
+		}
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   vers,
+		})
+		return
+	}
 
-	fmt.Printf("Project: %s Version: %s Stage: %s\n", prj, ver, stg)
 }