gobs webhook test

instead of anytime that it's not polled

README.md

@@ -14,11 +14,24 @@ You'll setup a project which will need the following information:
 2. Script to build the repository
 3. Script to test the repository
 4. Script to build the release file
-5. Path to the release file
+5. Path to the release file / can also be a script that returns a file name
 6. Script to set release name / version
 
+An optional set of environment strings can be set to define the environment in which the scripts run.
+```
+"environment": [
+"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/go/bin",
+"GOPATH=@dir"
+]
+```
+
 Projects will be defined in a project.json file for now.  I may add a web interface later.
 
+@dir in any of the script strings or environment entries will be replaced with an absolute path to the current working directory of the specific version being worked on.
+```
+sh ./build.sh @dir
+```
+
 
 Ironsmith will take the information for the defined project above and do the following
 
@@ -26,7 +39,8 @@ Ironsmith will take the information for the defined project above and do the fol
 2. Change to that directory
 2. Create a bolt DB file for the project to keep a log of all the builds
 3. Run an initial pull of the repository using the pull script
-4. If pull succeeds, Run the Build Scripts
+4. Run version script
+4. If pull is a new version, then Run the Build Scripts
 5. If build succeeds, run the test scripts
 6. If test succeeds, run the release scripts
 7. Load the release file into project release folder with the release name

build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+go-bindata web/... && go build -a -o ironsmith

cycle.go

@@ -7,88 +7,17 @@ package main
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
-	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
-	"git.townsourced.com/ironsmith/datastore"
+	"git.townsourced.com/townsourced/ironsmith/datastore"
 )
 
-func (p *Project) errHandled(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	if p.ds == nil {
-		log.Printf("Error in project %s: %s\n", p.id(), err)
-		return true
-	}
-	defer func() {
-		err = p.ds.Close()
-		if err != nil {
-			log.Printf("Error closing the datastore for project %s: %s\n", p.id(), err)
-		}
-		p.ds = nil
-
-		//clean up version folder if it exists
-
-		if p.version != "" {
-			err = os.RemoveAll(p.verDir())
-			log.Printf("Error deleting the version directory project %s version %s: %s\n",
-				p.id(), p.version, err)
-
-		}
-	}()
-
-	lerr := p.ds.AddLog(p.version, p.stage, err.Error())
-	if lerr != nil {
-		log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s",
-			p.id(), err, lerr)
-	}
-
-	return true
-}
-
-func (p *Project) id() string {
-	if p.filename == "" {
-		panic("invalid project filename")
-	}
-	return strings.TrimSuffix(p.filename, filepath.Ext(p.filename))
-}
-
-func (p *Project) dir() string {
-	return filepath.Join(dataDir, p.id())
-}
-
-func (p *Project) verDir() string {
-	return filepath.Join(p.dir(), p.version)
-}
-
-// prepData makes sure the project's data folder and data store is created
-/*
-	folder structure
-	projectDataFolder/<project-name>/<project-version>
-
-*/
-func (p *Project) prepData() error {
-	err := os.MkdirAll(p.dir(), 0777)
-	if err != nil {
-		return err
-	}
-
-	p.ds, err = datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith"))
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 /*
 Project life cycle:
 (Load Project file) -> (Fetch) -> (Build) -> (Test) -> (Release) - > (Sleep for polling period) ->
@@ -98,19 +27,28 @@ Project life cycle:
 
 // load is the beginning of the cycle.  Loads / reloads the project file to make sure that the scripts are up-to-date
 // call's fetch and triggers the next poll if one exists
-func (p *Project) load() {
-	p.version = ""
+func (p *Project) load(forceBuild bool) {
+	p.processing.Lock() // ensure only one cycle is running at a time per project
+	defer p.processing.Unlock()
+
+	p.setStage(stageLoad)
+	p.setVersion("Version not yet set")
+	p.start = time.Time{}
 
 	if p.filename == "" {
 		p.errHandled(errors.New("Invalid project file name"))
 		return
 	}
 
-	if !projects.exists(p.filename) {
+	if _, ok := projects.get(p.id()); !ok {
 		// project has been deleted
 		// don't continue polling
-		// move project data to deleted folder
-		p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, p.id())))
+		// move project data to deleted folder with a timestamp
+		if p.errHandled(p.close()) {
+			return
+		}
+		p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir,
+			strconv.FormatInt(time.Now().Unix(), 10), p.id())))
 		return
 	}
 
@@ -119,102 +57,113 @@ func (p *Project) load() {
 		return
 	}
 
-	if p.errHandled(json.Unmarshal(data, p)) {
+	new := &Project{}
+	if p.errHandled(json.Unmarshal(data, new)) {
 		return
 	}
 
-	p.stage = stageLoad
+	p.setData(new)
 
-	if p.errHandled(p.prepData()) {
-		return
-	}
+	p.fetch(forceBuild)
 
-	if p.PollInterval != "" {
-		p.poll, err = time.ParseDuration(p.PollInterval)
-		if p.errHandled(err) {
-			p.poll = 0
-		}
-	}
-
-	p.fetch()
+	p.setStage(stageWait)
 
 	//full cycle completed
+	p.errHandled(p.ds.TrimVersions(p.MaxVersions))
 
 	if p.poll > 0 {
 		//start polling
-		time.AfterFunc(p.poll, p.load)
+		time.AfterFunc(p.poll, func() {
+			p.load(false)
+		})
 	}
 }
 
-// fetch first runs the version script and checks the returned version against the latest version in the
-// project database. If the version hasn't changed, then it breaks out of the cycle early doing nothing
-// if the version has changed, then it runs the fetch script
-func (p *Project) fetch() {
-	p.stage = stageFetch
-	verCmd := &exec.Cmd{
-		Path: p.Version,
-		Dir:  p.dir(),
-	}
+// fetch first runs the fetch script into a temporary directory
+// then it runs the version script in the temp directory to see if there is a newer version of the
+// fetched code, if there is then the temp dir is renamed to the version name
+func (p *Project) fetch(forceBuild bool) {
+	p.setStage(stageFetch)
+	p.start = time.Now()
 
-	version, err := verCmd.Output()
-
-	if p.errHandled(err) {
+	if p.Fetch == "" {
 		return
 	}
 
-	p.version = string(version)
+	tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10))
 
-	lVer, err := p.ds.LatestVersion()
-	if err != datastore.ErrNotFound && p.errHandled(err) {
-		return
-	}
-
-	if p.version == lVer {
-		// no new build
-		return
-	}
-
-	if p.errHandled(os.MkdirAll(p.verDir(), 0777)) {
+	if p.errHandled(os.MkdirAll(tempDir, 0777)) {
 		return
 	}
 
 	//fetch project
-	fetchCmd := &exec.Cmd{
-		Path: p.Fetch,
-		Dir:  p.verDir(),
-	}
-
-	fetchResult, err := fetchCmd.Output()
+	fetchResult, err := runCmd(p.Fetch, tempDir, p.Environment)
 	if p.errHandled(err) {
 		return
 	}
 
-	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(fetchResult))) {
+	// fetched succesfully, determine version
+	version, err := runCmd(p.Version, tempDir, p.Environment)
+
+	if p.errHandled(err) {
 		return
 	}
 
-	// fetched succesfully, onto the build stage
-	p.build()
+	p.setVersion(strings.TrimSpace(string(version)))
 
+	if !forceBuild {
+		// if not forced build, then check if this specific version has attempted a build yet
+		lVer, err := p.ds.LastVersion(stageBuild)
+		if err != datastore.ErrNotFound && p.errHandled(err) {
+			return
+		}
+
+		if p.version == "" || p.version == lVer.Version {
+			// no new build clean up temp dir
+			p.errHandled(os.RemoveAll(tempDir))
+
+			vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
+			return
+		}
+	}
+
+	//remove any existing data that matches version hash
+	if p.errHandled(os.RemoveAll(p.workingDir())) {
+		return
+	}
+
+	//new version move tempdir to workingDir
+	if p.errHandled(os.Rename(tempDir, p.workingDir())) {
+		// cleanup temp dir if rename failed
+		p.errHandled(os.RemoveAll(tempDir))
+		return
+	}
+
+	//log fetch results
+	if p.errHandled(p.ds.AddLog(p.version, p.stage, string(fetchResult))) {
+		return
+	}
+
+	// continue to build
+	p.build()
 }
 
 // build  runs the build scripts to build the project which should result in the a single file
 // configured in the ReleaseFile section of the project file
 func (p *Project) build() {
-	p.stage = stageBuild
+	p.setStage(stageBuild)
 
-	buildCmd := &exec.Cmd{
-		Path: p.Build,
-		Dir:  p.verDir(),
+	if p.Build == "" {
+		return
 	}
 
-	output, err := buildCmd.Output()
+	output, err := runCmd(p.Build, p.workingDir(), p.Environment)
 
 	if p.errHandled(err) {
 		return
 	}
 
-	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(output))) {
+	if p.errHandled(p.ds.AddLog(p.version, p.stage, string(output))) {
 		return
 	}
 
@@ -224,49 +173,45 @@ func (p *Project) build() {
 
 // test runs the test scripts
 func (p *Project) test() {
-	p.stage = stageTest
+	p.setStage(stageTest)
 
-	testCmd := &exec.Cmd{
-		Path: p.Test,
-		Dir:  p.verDir(),
+	if p.Test == "" {
+		return
 	}
-
-	output, err := testCmd.Output()
+	output, err := runCmd(p.Test, p.workingDir(), p.Environment)
 
 	if p.errHandled(err) {
 		return
 	}
 
-	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(output))) {
+	if p.errHandled(p.ds.AddLog(p.version, p.stage, string(output))) {
 		return
 	}
 
 	//  Tests passed, onto release
 	p.release()
-
 }
 
 // release runs the release scripts and builds the release file
 func (p *Project) release() {
-	p.stage = stageRelease
+	p.setStage(stageRelease)
 
-	releaseCmd := &exec.Cmd{
-		Path: p.Release,
-		Dir:  p.verDir(),
+	if p.Release == "" {
+		return
 	}
 
-	output, err := releaseCmd.Output()
+	output, err := runCmd(p.Release, p.workingDir(), p.Environment)
 
 	if p.errHandled(err) {
 		return
 	}
 
-	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(output))) {
+	if p.errHandled(p.ds.AddLog(p.version, p.stage, string(output))) {
 		return
 	}
 
 	//get release file
-	f, err := os.Open(filepath.Join(p.verDir(), p.ReleaseFile))
+	f, err := os.Open(filepath.Join(p.workingDir(), p.ReleaseFile))
 	if p.errHandled(err) {
 		return
 	}
@@ -276,8 +221,20 @@ func (p *Project) release() {
 		return
 	}
 
-	if p.errHandled(p.ds.AddRelease(p.version, buff)) {
+	if p.errHandled(p.ds.AddRelease(p.version, filepath.Base(p.ReleaseFile), buff)) {
 		return
 	}
 
+	p.setStage(stageReleased)
+
+	if p.errHandled(p.ds.AddLog(p.version, p.stage,
+		fmt.Sprintf("Project %s Version %s built, tested, and released successfully and took %s.\n", p.id(), p.version,
+			time.Now().Sub(p.start)))) {
+		return
+	}
+
+	//build successfull, remove working dir
+	p.errHandled(os.RemoveAll(p.workingDir()))
+
+	vlog("Project %s Version %s built, tested, and released successfully.\n", p.id(), p.version)
 }

datastore/ds.go

@@ -6,17 +6,17 @@
 package datastore
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
-	"io"
 	"time"
 
 	"github.com/boltdb/bolt"
 )
 
+//TODO: Move this all over to GobStore if I ever get around to finishing it
+
 // ErrNotFound is the error returned when a value cannot be found in the store for the given key
-var ErrNotFound = errors.New("Value not found")
+var ErrNotFound = errors.New("Value not found in datastore")
 
 // Store is a datastore for getting and setting data for a given ironsmith project
 // run on top of a Bolt DB file
@@ -64,31 +64,99 @@ func Open(filename string) (*Store, error) {
 
 // Close closes the bolt datastore
 func (ds *Store) Close() error {
-	return ds.Close()
+	return ds.bolt.Close()
 }
 
-func (ds *Store) get(bucket string, key TimeKey, result interface{}) error {
-	return ds.bolt.View(func(tx *bolt.Tx) error {
-		dsValue := tx.Bucket([]byte(bucket)).Get(key.Bytes())
+// TrimVersions Removes versions from the datastore file until it reaches the maxVersions count
+func (ds *Store) TrimVersions(maxVersions int) error {
+	if maxVersions <= 0 {
+		// no max set
+		return nil
+	}
 
-		if dsValue == nil {
-			return ErrNotFound
+	versions, err := ds.Versions()
+	if err != nil {
+		return err
+	}
+
+	if len(versions) <= maxVersions {
+		return nil
+	}
+
+	remove := versions[maxVersions:]
+
+	for i := range remove {
+		err = ds.deleteVersion(remove[i].Version)
+		if err != nil {
+			return err
 		}
+	}
 
-		if value, ok := result.([]byte); ok {
-			buff := bytes.NewBuffer(value)
-			_, err := io.Copy(buff, bytes.NewReader(dsValue))
+	return nil
+}
+
+// removes the earliest instance of a specific version
+func (ds *Store) deleteVersion(version string) error {
+	return ds.bolt.Update(func(tx *bolt.Tx) error {
+		// remove all logs for this version
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			lg := &Log{}
+
+			err := json.Unmarshal(v, lg)
 			if err != nil {
 				return err
 			}
+
+			if lg.Version != version {
+				break
+			}
+
+			err = c.Delete()
+			if err != nil {
+				return err
+			}
+		}
+
+		// remove all releases for this version
+		release, err := ds.Release(version)
+		if err == ErrNotFound {
 			return nil
 		}
+		if err != nil {
+			return err
+		}
+
+		err = tx.Bucket([]byte(bucketReleases)).Delete(release.FileKey.Bytes())
+		if err != nil {
+			return err
+		}
+
+		// remove release file for this version
+		err = tx.Bucket([]byte(bucketFiles)).Delete(release.FileKey.Bytes())
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+}
+
+func (ds *Store) get(bucket string, key []byte, result interface{}) error {
+	return ds.bolt.View(func(tx *bolt.Tx) error {
+		dsValue := tx.Bucket([]byte(bucket)).Get(key)
+
+		if dsValue == nil {
+			return ErrNotFound
+		}
 
 		return json.Unmarshal(dsValue, result)
 	})
 }
 
-func (ds *Store) put(bucket string, key TimeKey, value interface{}) error {
+func (ds *Store) put(bucket string, key []byte, value interface{}) error {
 	var err error
 	dsValue, ok := value.([]byte)
 	if !ok {
@@ -99,12 +167,6 @@ func (ds *Store) put(bucket string, key TimeKey, value interface{}) error {
 	}
 
 	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		return tx.Bucket([]byte(bucket)).Put(key.Bytes(), dsValue)
-	})
-}
-
-func (ds *Store) delete(bucket string, key TimeKey) error {
-	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		return tx.Bucket([]byte(bucket)).Delete(key.Bytes())
+		return tx.Bucket([]byte(bucket)).Put(key, dsValue)
 	})
 }

datastore/key.go

@@ -29,10 +29,6 @@ func NewTimeKey() TimeKey {
 	nsec := t.Nanosecond()
 
 	return TimeKey{
-		rBits[0], //random
-		rBits[1],
-		rBits[2],
-		rBits[3],
 		byte(sec >> 56), // seconds
 		byte(sec >> 48),
 		byte(sec >> 40),
@@ -45,12 +41,16 @@ func NewTimeKey() TimeKey {
 		byte(nsec >> 16),
 		byte(nsec >> 8),
 		byte(nsec),
+		rBits[0], //random
+		rBits[1],
+		rBits[2],
+		rBits[3],
 	}
 }
 
 // Time returns the time portion of a timekey
 func (k TimeKey) Time() time.Time {
-	buf := k[4:]
+	buf := k[:]
 
 	sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 |
 		int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56
@@ -83,3 +83,44 @@ func (k TimeKey) UUID() string {
 func (k TimeKey) Bytes() []byte {
 	return []byte(k[:])
 }
+
+// String returns the string representation of a timekey
+func (k TimeKey) String() string {
+	return k.UUID()
+}
+
+// MarshalJSON implements JSON marshaler
+func (k *TimeKey) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + k.String() + `"`), nil
+}
+
+// UnmarshalJSON implements JSON unmarshaler
+func (k *TimeKey) UnmarshalJSON(buf []byte) error {
+	// drop quotes
+	buf = buf[1 : len(buf)-1]
+	_, err := hex.Decode(k[0:4], buf[0:8])
+	if err != nil {
+		return err
+	}
+
+	_, err = hex.Decode(k[4:6], buf[9:13])
+	if err != nil {
+		return err
+	}
+
+	_, err = hex.Decode(k[6:8], buf[14:18])
+	if err != nil {
+		return err
+	}
+
+	_, err = hex.Decode(k[8:10], buf[19:23])
+	if err != nil {
+		return err
+	}
+	_, err = hex.Decode(k[10:], buf[24:])
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

datastore/log.go

@@ -11,11 +11,12 @@ import (
 	"github.com/boltdb/bolt"
 )
 
-type log struct {
-	When    time.Time `json:"when"`
-	Version string    `json:"version"`
-	Stage   string    `json:"stage"`
-	Log     string    `json:"log"`
+// Log is a version log entry for a project
+type Log struct {
+	When    time.Time `json:"when,omitempty"`
+	Version string    `json:"version,omitempty"`
+	Stage   string    `json:"stage,omitempty"`
+	Log     string    `json:"log,omitempty"`
 }
 
 const bucketLog = "log"
@@ -24,41 +25,155 @@ const bucketLog = "log"
 func (ds *Store) AddLog(version, stage, entry string) error {
 	key := NewTimeKey()
 
-	data := &log{
+	data := &Log{
 		When:    key.Time(),
 		Version: version,
 		Stage:   stage,
 		Log:     entry,
 	}
 
-	return ds.put(bucketLog, key, data)
+	return ds.put(bucketLog, key.Bytes(), data)
 }
 
-// LatestVersion returns the latest version for the current project
-func (ds *Store) LatestVersion() (string, error) {
-	version := ""
+// LastVersion returns the last version in the log for the given stage.  If stage is blank,
+// then it returns the last of any stage
+func (ds *Store) LastVersion(stage string) (*Log, error) {
+	last := &Log{}
 
 	err := ds.bolt.View(func(tx *bolt.Tx) error {
 		c := tx.Bucket([]byte(bucketLog)).Cursor()
 
 		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			l := &log{}
+			l := &Log{}
 			err := json.Unmarshal(v, l)
 			if err != nil {
 				return err
 			}
 
 			if l.Version != "" {
-				version = l.Version
+				if stage == "" || l.Stage == stage {
+					last = l
+					return nil
+				}
+			}
+		}
+
+		return nil // not found return blank
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return last, nil
+}
+
+// Versions lists the versions in a given project, including the last stage that version got to
+func (ds *Store) Versions() ([]*Log, error) {
+	var vers []*Log
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		var current = ""
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			l := &Log{}
+			err := json.Unmarshal(v, l)
+			if err != nil {
+				return err
+			}
+
+			// capture the newest entry for each version
+			if l.Version != current {
+				vers = append(vers, l)
+				current = l.Version
+			}
+
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return vers, nil
+}
+
+// VersionLog returns all the log entries for a given version
+func (ds *Store) VersionLog(version string) ([]*Log, error) {
+	var logs []*Log
+
+	if version == "" {
+		return logs, nil
+	}
+
+	verFound := false
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			l := &Log{}
+			err := json.Unmarshal(v, l)
+			if err != nil {
+				return err
+			}
+
+			if verFound && l.Version != version {
 				return nil
 			}
+
+			if l.Version == version {
+				logs = append(logs, l)
+				verFound = true
+			}
+
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return logs, nil
+}
+
+// StageLog returns the log entry for a given version + stage
+func (ds *Store) StageLog(version, stage string) (*Log, error) {
+	var entry *Log
+
+	if version == "" || stage == "" {
+		return nil, ErrNotFound
+	}
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			l := &Log{}
+			err := json.Unmarshal(v, l)
+			if err != nil {
+				return err
+			}
+
+			if l.Version == version && l.Stage == stage {
+				entry = l
+				return nil
+			}
+
 		}
 
 		return ErrNotFound
 	})
 
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	return version, nil
+
+	return entry, nil
 }

datastore/releases.go

@@ -5,16 +5,20 @@
 package datastore
 
 import (
+	"bytes"
 	"encoding/json"
+	"io"
 	"time"
 
 	"github.com/boltdb/bolt"
 )
 
-type release struct {
-	When    time.Time `json:"when"`
-	Version string    `json:"version"`
-	FileKey TimeKey   `json:"file"`
+// Release is a record of the fully built and ready to deploy release file
+type Release struct {
+	When     time.Time `json:"when"`
+	Version  string    `json:"version"`
+	FileName string    `json:"fileName"`
+	FileKey  TimeKey   `json:"fileKey"`
 }
 
 const (
@@ -23,14 +27,14 @@ const (
 )
 
 // AddRelease adds a new Release
-func (ds *Store) AddRelease(version string, fileData []byte) error {
+func (ds *Store) AddRelease(version, fileName string, fileData []byte) error {
 	key := NewTimeKey()
-	fileKey := NewTimeKey()
 
-	r := &release{
-		When:    key.Time(),
-		Version: version,
-		FileKey: fileKey,
+	r := &Release{
+		When:     key.Time(),
+		Version:  version,
+		FileName: fileName,
+		FileKey:  key,
 	}
 
 	dsValue, err := json.Marshal(r)
@@ -44,6 +48,111 @@ func (ds *Store) AddRelease(version string, fileData []byte) error {
 			return err
 		}
 
-		return tx.Bucket([]byte(bucketFiles)).Put(fileKey.Bytes(), fileData)
+		return tx.Bucket([]byte(bucketFiles)).Put(key.Bytes(), fileData)
 	})
 }
+
+// ReleaseFile returns a specific file from a release for the given file key
+func (ds *Store) ReleaseFile(fileKey TimeKey) ([]byte, error) {
+	var fileData bytes.Buffer
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		dsValue := tx.Bucket([]byte(bucketFiles)).Get(fileKey.Bytes())
+
+		if dsValue == nil {
+			return ErrNotFound
+		}
+
+		_, err := io.Copy(&fileData, bytes.NewReader(dsValue))
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return fileData.Bytes(), nil
+}
+
+// Release gets the release record for a specific version
+func (ds *Store) Release(version string) (*Release, error) {
+	r := &Release{}
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketReleases)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			err := json.Unmarshal(v, r)
+			if err != nil {
+				return err
+			}
+
+			if r.Version == version {
+				return nil
+			}
+		}
+
+		return ErrNotFound
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}
+
+// Releases lists all the releases in a given project
+func (ds *Store) Releases() ([]*Release, error) {
+	var vers []*Release
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketReleases)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			r := &Release{}
+			err := json.Unmarshal(v, r)
+			if err != nil {
+				return err
+			}
+
+			vers = append(vers, r)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return vers, nil
+}
+
+// LastRelease lists the last release for a project
+func (ds *Store) LastRelease() (*Release, error) {
+	r := &Release{}
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		_, v := tx.Bucket([]byte(bucketReleases)).Cursor().Last()
+
+		if v == nil {
+			return ErrNotFound
+		}
+
+		err := json.Unmarshal(v, r)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}

error.go

@@ -0,0 +1,147 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+
+	"git.townsourced.com/townsourced/ironsmith/datastore"
+)
+
+const (
+	acceptHTML = "text/html"
+)
+
+// Err404 is a standard 404 error response
+var Err404 = errors.New("Resource not found")
+
+func errHandled(err error, w http.ResponseWriter, r *http.Request) bool {
+	if err == nil {
+		return false
+	}
+
+	if err == datastore.ErrNotFound {
+		four04(w, r)
+		return true
+	}
+
+	var status, errMsg string
+
+	errMsg = err.Error()
+
+	switch err.(type) {
+
+	case *Fail:
+		status = statusFail
+	case *http.ProtocolError, *json.SyntaxError, *json.UnmarshalTypeError:
+		//Hardcoded external errors which can bubble up to the end users
+		// without exposing internal server information, make them failures
+		err = FailFromErr(err)
+		status = statusFail
+
+		errMsg = fmt.Sprintf("We had trouble parsing your input, please check your input and try again: %s", err)
+	default:
+		status = statusError
+		log.Printf("An error has occurred from a web request: %s", errMsg)
+		errMsg = "An internal server error has occurred"
+	}
+
+	if status == statusFail {
+		respondJsendCode(w, &JSend{
+			Status:  status,
+			Message: errMsg,
+			Data:    err.(*Fail).Data,
+		}, err.(*Fail).HTTPStatus)
+	} else {
+		respondJsend(w, &JSend{
+			Status:  status,
+			Message: errMsg,
+		})
+	}
+
+	return true
+}
+
+// four04 is a standard 404 response if request header accepts text/html
+// they'll get a 404 page, otherwise a json response
+func four04(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Content-Type", "application/json")
+
+	response := &JSend{
+		Status:  statusFail,
+		Message: "Resource not found",
+		Data:    r.URL.String(),
+	}
+
+	w.WriteHeader(http.StatusNotFound)
+
+	result, err := json.Marshal(response)
+	if err != nil {
+		log.Printf("Error marshalling 404 response: %s", err)
+		return
+	}
+
+	_, err = w.Write(result)
+	if err != nil {
+		log.Printf("Error in four04: %s", err)
+	}
+}
+
+// Fail is an error whose contents can be exposed to the client and is usually the result
+// of incorrect client input
+type Fail struct {
+	Message    string      `json:"message,omitempty"`
+	Data       interface{} `json:"data,omitempty"`
+	HTTPStatus int         `json:"-"` //gets set in the error response
+}
+
+func (f *Fail) Error() string {
+	return f.Message
+}
+
+// NewFail creates a new failure, data is optional
+func NewFail(message string, data ...interface{}) error {
+	return &Fail{
+		Message:    message,
+		Data:       data,
+		HTTPStatus: 0,
+	}
+}
+
+// FailFromErr returns a new failure based on the passed in error, data is optional
+// if passed in error is nil, then nil is returned
+func FailFromErr(err error, data ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+	return NewFail(err.Error(), data...)
+}
+
+// IsEqual tests whether an error is equal to another error / failure
+func (f *Fail) IsEqual(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	return err.Error() == f.Error()
+}
+
+// IsFail tests whether the passed in error is a failure
+func IsFail(err error) bool {
+	if err == nil {
+		return false
+	}
+	switch err.(type) {
+	case *Fail:
+		return true
+	default:
+		return false
+	}
+}

exec.go

@@ -0,0 +1,98 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func runCmd(cmd, dir string, env []string) ([]byte, error) {
+	s := strings.Fields(strings.Replace(cmd, "@dir", dir, -1))
+
+	for i := range env {
+		env[i] = strings.Replace(env[i], "@dir", dir, -1)
+	}
+
+	var args []string
+
+	if len(s) > 1 {
+		args = s[1:]
+	}
+
+	name := s[0]
+	ec := &exec.Cmd{
+		Path: name,
+		Args: append([]string{name}, args...),
+		Dir:  dir,
+		Env:  env,
+	}
+
+	if filepath.Base(name) == name {
+		lp, err := lookPath(name, env)
+		if err != nil {
+			return nil, err
+		}
+		ec.Path = lp
+	}
+
+	vlog("Executing command: %s in dir %s\n", cmd, dir)
+
+	result, err := ec.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("%s\n%s", err, result)
+	}
+	return result, nil
+}
+
+// similar to os/exec.LookPath, except it checks if the passed in
+// custom environment includes a path definitions and uses that path instead
+// note this probably only works on unix, that's all I care about for now
+func lookPath(file string, env []string) (string, error) {
+	if strings.Contains(file, "/") {
+		err := findExecutable(file)
+		if err == nil {
+			return file, nil
+		}
+		return "", &exec.Error{Name: file, Err: err}
+	}
+
+	for i := range env {
+		if strings.HasPrefix(env[i], "PATH=") {
+			pathenv := env[i][5:]
+			if pathenv == "" {
+				return "", &exec.Error{Name: file, Err: exec.ErrNotFound}
+			}
+			for _, dir := range strings.Split(pathenv, ":") {
+				if dir == "" {
+					// Unix shell semantics: path element "" means "."
+					dir = "."
+				}
+				path := dir + "/" + file
+				if err := findExecutable(path); err == nil {
+					return path, nil
+				}
+			}
+			return "", &exec.Error{Name: file, Err: exec.ErrNotFound}
+		}
+	}
+
+	return exec.LookPath(file)
+}
+
+func findExecutable(file string) error {
+	d, err := os.Stat(file)
+	if err != nil {
+		return err
+	}
+	if m := d.Mode(); !m.IsDir() && m&0111 != 0 {
+		return nil
+	}
+
+	return os.ErrPermission
+}

json.go

@@ -0,0 +1,100 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+)
+
+const (
+	statusSuccess = "success"
+	statusError   = "error"
+	statusFail    = "fail"
+)
+
+const maxJSONSize = 1 << 20 //10MB
+
+var errInputTooLarge = &Fail{
+	Message:    "Input size is too large, please check your input and try again",
+	HTTPStatus: http.StatusRequestEntityTooLarge,
+}
+
+// JSend is the standard format for a response from townsourced
+type JSend struct {
+	Status   string      `json:"status"`
+	Data     interface{} `json:"data,omitempty"`
+	Message  string      `json:"message,omitempty"`
+	Failures []error     `json:"failures,omitempty"`
+	More     bool        `json:"more,omitempty"` // more data exists for this request
+}
+
+//respondJsend marshalls the input into a json byte array
+// and writes it to the reponse with appropriate header
+func respondJsend(w http.ResponseWriter, response *JSend) {
+	respondJsendCode(w, response, 0)
+}
+
+// respondJsendCode is the same as respondJSend, but lets you specify a status code
+func respondJsendCode(w http.ResponseWriter, response *JSend, statusCode int) {
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Content-Type", "application/json")
+
+	if len(response.Failures) > 0 && response.Message == "" {
+		response.Message = "One or more item has failed. Check the individual failures for details."
+	}
+
+	result, err := json.MarshalIndent(response, "", "    ")
+	if err != nil {
+		log.Printf("Error marshalling response: %s", err)
+
+		result, _ = json.Marshal(&JSend{
+			Status:  statusError,
+			Message: "An internal error occurred, and we'll look into it.",
+		})
+	}
+
+	if statusCode <= 0 {
+		switch response.Status {
+		case statusFail:
+			w.WriteHeader(http.StatusBadRequest)
+		case statusError:
+			w.WriteHeader(http.StatusInternalServerError)
+		}
+		//default is status 200
+	} else {
+		w.WriteHeader(statusCode)
+	}
+
+	_, err = w.Write(result)
+	if err != nil {
+		log.Printf("Error writing jsend response: %s", err)
+	}
+}
+
+func parseInput(r *http.Request, result interface{}) error {
+	lr := &io.LimitedReader{R: r.Body, N: maxJSONSize + 1}
+	buff, err := ioutil.ReadAll(lr)
+	if err != nil {
+		return err
+	}
+
+	if lr.N == 0 {
+		return errInputTooLarge
+	}
+
+	if len(buff) == 0 {
+		return nil
+	}
+
+	err = json.Unmarshal(buff, result)
+	if err != nil {
+		return err
+	}
+	return nil
+}

log.go

@@ -0,0 +1,13 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import "log"
+
+func vlog(format string, v ...interface{}) {
+	if verbose {
+		log.Printf(format, v...)
+	}
+}

main.go

@@ -5,34 +5,59 @@
 package main
 
 import (
+	"flag"
 	"log"
 	"os"
+	"os/signal"
 	"path/filepath"
 
-	"git.townsourced.com/config"
+	"git.townsourced.com/townsourced/config"
 )
 
 //settings
 var (
 	projectDir = "./projects" // /etc/ironsmith/
 	dataDir    = "./data"     // /var/ironsmith/
-	address    = "http://localhost:8026"
+	address    = ":8026"
 	certFile   = ""
 	keyFile    = ""
 )
 
+//flags
+var (
+	verbose = false
+)
+
+func init() {
+	flag.BoolVar(&verbose, "v", false, "Verbose prints to stdOut every command and stage as it processes")
+
+	//Capture program shutdown, to make sure everything shuts down nicely
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		for sig := range c {
+			if sig == os.Interrupt {
+				projects.stopAll()
+				os.Exit(0)
+			}
+		}
+	}()
+}
+
 func main() {
+	flag.Parse()
+
 	settingPaths := config.StandardFileLocations("ironsmith/settings.json")
-	log.Println("IronSmith will use settings files in the following locations (in order of priority):")
+	vlog("IronSmith will use settings files in the following locations (in order of priority):\n")
 	for i := range settingPaths {
-		log.Println("\t", settingPaths[i])
+		vlog("\t%s\n", settingPaths[i])
 	}
 	cfg, err := config.LoadOrCreate(settingPaths...)
 	if err != nil {
 		log.Fatalf("Error loading or creating IronSmith settings file: %s", err)
 	}
 
-	log.Printf("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
+	vlog("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
 
 	projectDir = cfg.String("projectDir", projectDir)
 	dataDir = cfg.String("dataDir", dataDir)
@@ -40,6 +65,9 @@ func main() {
 	certFile = cfg.String("certFile", certFile)
 	keyFile = cfg.String("keyFile", keyFile)
 
+	vlog("Project Definition Directory: %s\n", projectDir)
+	vlog("Project Data Directory: %s\n", dataDir)
+
 	//prep dirs
 	err = os.MkdirAll(filepath.Join(projectDir, enabledProjectDir), 0777)
 	if err != nil {
@@ -61,5 +89,11 @@ func main() {
 	if err != nil {
 		log.Fatalf("Error loading projects: %s", err)
 	}
-	//start server
+
+	//start web server
+	err = startServer()
+	if err != nil {
+		log.Fatalf("Error Starting web server: %s", err)
+	}
+
 }

project.go

@@ -5,14 +5,17 @@
 package main
 
 import (
+	"crypto/sha1"
 	"encoding/json"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
-	"git.townsourced.com/ironsmith/datastore"
+	"git.townsourced.com/townsourced/ironsmith/datastore"
 )
 
 const (
@@ -22,11 +25,13 @@ const (
 
 //stages
 const (
-	stageLoad    = "load"
-	stageFetch   = "fetch"
-	stageBuild   = "build"
-	stageTest    = "test"
-	stageRelease = "release"
+	stageLoad     = "loading"
+	stageFetch    = "fetching"
+	stageBuild    = "building"
+	stageTest     = "testing"
+	stageRelease  = "releasing"
+	stageReleased = "released"
+	stageWait     = "waiting"
 )
 
 const projectFilePoll = 30 * time.Second
@@ -43,6 +48,8 @@ The project lifecycle goes like this, each step calling the next if successful
 type Project struct {
 	Name string `json:"name"` // name of the project
 
+	Environment []string `json:"environment"` // Environment for each of the scripts below, if empty will use the current processes environment
+
 	Fetch   string `json:"fetch"`   //Script to fetch the latest project code into the current directory
 	Build   string `json:"build"`   //Script to build the latest project code
 	Test    string `json:"test"`    //Script to test the latest project code
@@ -51,14 +58,260 @@ type Project struct {
 	Version string `json:"version"` //Script to generate the version num of the current build, should be indempotent
 
 	ReleaseFile   string `json:"releaseFile"`
-	PollInterval  string `json:"pollInterval"`  // if not poll interval is specified, this project is trigger only
-	TriggerSecret string `json:"triggerSecret"` //secret to be included with a trigger call
+	PollInterval  string `json:"pollInterval,omitempty"`  // if not poll interval is specified, this project is trigger only
+	TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call
+	MaxVersions   int    `json:"maxVersions,omitempty"`   // Max number of versions to keep in the project datastore
 
 	filename string
 	poll     time.Duration
 	ds       *datastore.Store
 	stage    string
+	status   string
 	version  string
+	hash     string
+	start    time.Time // the last start time of the latest cycle
+
+	sync.RWMutex
+	processing sync.Mutex
+}
+
+func (p *Project) errHandled(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	vlog("Error in project %s: %s\n", p.id(), err)
+
+	if p.ds == nil {
+		log.Printf("Error in project %s: %s\n", p.id(), err)
+		return true
+	}
+	defer func() {
+		//clean up version folder if it exists
+		if p.version != "" {
+			err = os.RemoveAll(p.workingDir())
+			if err != nil {
+				log.Printf("Error deleting the version directory project %s version %s: %s\n",
+					p.id(), p.version, err)
+			}
+
+		}
+	}()
+
+	lerr := p.ds.AddLog(p.version, p.stage, err.Error())
+	if lerr != nil {
+		log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s",
+			p.id(), err, lerr)
+	}
+
+	return true
+}
+
+func projectID(filename string) string {
+	return strings.TrimSuffix(filename, filepath.Ext(filename))
+}
+
+func (p *Project) id() string {
+	if p.filename == "" {
+		panic("invalid project filename")
+	}
+	return projectID(p.filename)
+}
+
+func (p *Project) dir() string {
+	return filepath.Join(dataDir, p.id())
+}
+
+func (p *Project) workingDir() string {
+	if p.hash == "" {
+		panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
+	}
+
+	//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
+	// timestamp on instead would work just fine, but I like having the working dir tied directly to the
+	// version returned by project script
+
+	return filepath.Join(p.dir(), p.hash)
+}
+
+// prepData makes sure the project's data folder and data store is created
+/*
+	folder structure
+	projectDataFolder/<project-name>/<project-version>
+
+*/
+func (p *Project) open() error {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.ds != nil {
+		return nil
+	}
+
+	err := os.MkdirAll(p.dir(), 0777)
+	if err != nil {
+		return err
+	}
+
+	ds, err := datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith"))
+	if err != nil {
+		return err
+	}
+
+	p.ds = ds
+
+	return nil
+}
+
+func (p *Project) setVersion(version string) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.version = version
+	if version == "" {
+		p.hash = ""
+		return
+	}
+
+	p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(version)))
+}
+
+func (p *Project) setStage(stage string) {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.version != "" {
+		vlog("Entering %s stage for Project: %s Version: %s\n", stage, p.id(), p.version)
+	} else {
+		vlog("Entering %s stage for Project: %s\n", stage, p.id())
+	}
+
+	p.stage = stage
+}
+
+type webProject struct {
+	ID             string         `json:"id"`
+	Name           string         `json:"name"`
+	ReleaseVersion string         `json:"releaseVersion"` //last successfully released version
+	Stage          string         `json:"stage"`          // current stage
+	LastLog        *datastore.Log `json:"lastLog"`
+}
+
+func (p *Project) webData() (*webProject, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	last, err := p.ds.LastVersion("")
+	if err != nil {
+		return nil, err
+	}
+
+	release, err := p.ds.LastVersion(stageRelease)
+	if err != nil {
+		return nil, err
+	}
+
+	d := &webProject{
+		Name:           p.Name,
+		ID:             p.id(),
+		ReleaseVersion: release.Version,
+		Stage:          p.stage,
+		LastLog:        last,
+	}
+
+	return d, nil
+}
+
+func (p *Project) versions() ([]*datastore.Log, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.Versions()
+}
+
+func (p *Project) versionLog(version string) ([]*datastore.Log, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.VersionLog(version)
+}
+
+func (p *Project) stageLog(version, stage string) (*datastore.Log, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.StageLog(version, stage)
+}
+
+func (p *Project) releases() ([]*datastore.Release, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.Releases()
+}
+
+func (p *Project) lastRelease() (*datastore.Release, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.LastRelease()
+}
+func (p *Project) releaseData(version string) (*datastore.Release, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.Release(version)
+}
+
+func (p *Project) releaseFile(fileKey datastore.TimeKey) ([]byte, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.ReleaseFile(fileKey)
+}
+
+// releaseFile
+
+func (p *Project) setData(new *Project) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.Name = new.Name
+	p.Environment = new.Environment
+
+	p.Fetch = new.Fetch
+	p.Build = new.Build
+	p.Test = new.Test
+	p.Release = new.Release
+	p.Version = new.Version
+
+	p.ReleaseFile = new.ReleaseFile
+	p.PollInterval = new.PollInterval
+	p.TriggerSecret = new.TriggerSecret
+	p.MaxVersions = new.MaxVersions
+
+	if p.PollInterval != "" {
+		var err error
+		p.poll, err = time.ParseDuration(p.PollInterval)
+		if p.errHandled(err) {
+			p.poll = 0
+		}
+	}
+}
+
+func (p *Project) close() error {
+	p.Lock()
+	defer p.Unlock()
+	if p.ds == nil {
+		return nil
+	}
+	err := p.ds.Close()
+	if err != nil {
+		return err
+	}
+
+	p.ds = nil
+	return nil
 }
 
 const projectTemplateFilename = "template.project.json"
@@ -66,19 +319,22 @@ const projectTemplateFilename = "template.project.json"
 var projectTemplate = &Project{
 	Name:    "Template Project",
 	Fetch:   "git clone root@git.townsourced.com:tshannon/ironsmith.git .",
-	Build:   "sh ./ironsmith/build.sh",
-	Test:    "sh ./ironsmith/test.sh",
-	Release: "sh ./ironsmith/release.sh",
+	Build:   "go build -a -v -o ironsmith",
+	Test:    "go test ./...",
+	Release: "tar -czf release.tar.gz ironsmith",
 	Version: "git describe --tags --long",
 
-	ReleaseFile:  `json:"./ironsmith/release.tar.gz"`,
+	ReleaseFile:  "release.tar.gz",
 	PollInterval: "15m",
+
+	MaxVersions: 100,
 }
 
 func prepTemplateProject() error {
 	filename := filepath.Join(projectDir, projectTemplateFilename)
 	_, err := os.Stat(filename)
 	if os.IsNotExist(err) {
+		vlog("Creating template project file in %s", filename)
 		f, err := os.Create(filename)
 		defer func() {
 			if cerr := f.Close(); cerr != nil && err == nil {
@@ -117,6 +373,7 @@ var projects = projectList{
 }
 
 func (p *projectList) load() error {
+	vlog("Loading projects from the enabled definitions in %s\n", filepath.Join(projectDir, enabledProjectDir))
 	dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
 	defer func() {
 		if cerr := dir.Close(); cerr != nil && err == nil {
@@ -144,15 +401,16 @@ func (p *projectList) load() error {
 	return nil
 }
 
-func (p *projectList) exists(name string) bool {
+func (p *projectList) get(name string) (*Project, bool) {
 	p.RLock()
 	defer p.RUnlock()
 
-	_, ok := p.data[name]
-	return ok
+	prj, ok := p.data[name]
+	return prj, ok
 }
 
 func (p *projectList) add(name string) {
+	vlog("Adding project %s to the project list.\n", name)
 	p.Lock()
 	defer p.Unlock()
 
@@ -161,10 +419,15 @@ func (p *projectList) add(name string) {
 		Name:     name,
 		stage:    stageLoad,
 	}
-	p.data[name] = prj
+	p.data[projectID(name)] = prj
 
 	go func() {
-		prj.load()
+		err := prj.open()
+		if err != nil {
+			log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err)
+			return
+		}
+		prj.load(false)
 	}()
 }
 
@@ -176,16 +439,48 @@ func (p *projectList) removeMissing(names []string) {
 	for i := range p.data {
 		found := false
 		for k := range names {
-			if names[k] == i {
+			if projectID(names[k]) == i {
 				found = true
 			}
 		}
 		if !found {
+			vlog("Removing project %s from the project list, because the project file was removed.\n",
+				i)
 			delete(p.data, i)
 		}
 	}
 }
 
+func (p *projectList) stopAll() {
+	p.RLock()
+	defer p.RUnlock()
+
+	for i := range p.data {
+		err := p.data[i].close()
+		if err != nil {
+			log.Printf("Error closing project datastore for Project: %s Error: %s\n", p.data[i].id(), err)
+		}
+	}
+}
+
+func (p *projectList) webList() ([]*webProject, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	list := make([]*webProject, 0, len(p.data))
+
+	for i := range p.data {
+		prj, err := p.data[i].webData()
+		if err != nil {
+			return nil, err
+		}
+
+		list = append(list, prj)
+	}
+
+	return list, nil
+}
+
 // startProjectLoader polls for new projects
 func startProjectLoader() {
 	dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
@@ -196,13 +491,13 @@ func startProjectLoader() {
 	}()
 
 	if err != nil {
-		log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", dir, err)
+		log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", projectDir, err)
 		return
 	}
 
 	files, err := dir.Readdir(0)
 	if err != nil {
-		log.Printf("Error in startProjectLoader reading the dir %s: %s\n", dir, err)
+		log.Printf("Error in startProjectLoader reading the dir %s: %s\n", projectDir, err)
 		return
 	}
 
@@ -211,7 +506,7 @@ func startProjectLoader() {
 	for i := range files {
 		if !files[i].IsDir() && filepath.Ext(files[i].Name()) == ".json" {
 			names[i] = files[i].Name()
-			if !projects.exists(files[i].Name()) {
+			if _, ok := projects.get(projectID(files[i].Name())); !ok {
 				projects.add(files[i].Name())
 			}
 		}

server.go

@@ -0,0 +1,146 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"net/http"
+	"path"
+	"time"
+)
+
+var webRoot *http.ServeMux
+
+func startServer() error {
+	var err error
+
+	routes()
+
+	server := &http.Server{
+		Handler: webRoot,
+		Addr:    address,
+	}
+
+	if certFile == "" || keyFile == "" {
+		err = server.ListenAndServe()
+	} else {
+		server.Addr = address
+		err = server.ListenAndServeTLS(certFile, keyFile)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type methodHandler struct {
+	get    http.HandlerFunc
+	post   http.HandlerFunc
+	put    http.HandlerFunc
+	delete http.HandlerFunc
+}
+
+func (m *methodHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if m.get == nil {
+		m.get = http.NotFound
+	}
+	if m.post == nil {
+		m.post = http.NotFound
+	}
+	if m.put == nil {
+		m.put = http.NotFound
+	}
+	if m.delete == nil {
+		m.delete = http.NotFound
+	}
+	switch r.Method {
+	case "GET":
+		m.get(w, r)
+		return
+	case "POST":
+		m.post(w, r)
+		return
+	case "PUT":
+		m.put(w, r)
+		return
+	case "DELETE":
+		m.delete(w, r)
+		return
+	default:
+		http.NotFound(w, r)
+		return
+	}
+}
+
+/*
+log Routes
+	/log/<project-id>/<version>/<stage>
+
+	/log/ - list all projects
+	/log/<project-id> - list all versions in a project, triggers new builds
+	/log/<project-id>/<version> - list combined output of all stages for a given version
+	/log/<project-id>/<version>/<stage> - list output of a given stage of a given version
+
+release routes
+	/release/<project-id>/<version>
+
+	/release/<project-id> - list last release for a given project  ?all returns all the releases for a project
+	/release/<project-id>/<version> - list release for a given project version
+
+trigger routes
+	/trigger/<project-id>
+		Triggers a project to start a cycle
+*/
+
+func routes() {
+	webRoot = http.NewServeMux()
+
+	webRoot.Handle("/", &methodHandler{
+		get: rootGet,
+	})
+
+	webRoot.Handle("/js/", &methodHandler{
+		get: assetGet,
+	})
+
+	webRoot.Handle("/css/", &methodHandler{
+		get: assetGet,
+	})
+
+	webRoot.Handle("/log/", &methodHandler{
+		get: logGet,
+	})
+
+	webRoot.Handle("/release/", &methodHandler{
+		get: releaseGet,
+	})
+
+	webRoot.Handle("/trigger/", &methodHandler{
+		post: triggerPost,
+	})
+
+}
+
+func rootGet(w http.ResponseWriter, r *http.Request) {
+	//send index.html
+	serveAsset(w, r, "web/index.html")
+}
+
+func assetGet(w http.ResponseWriter, r *http.Request) {
+	serveAsset(w, r, path.Join("web", r.URL.Path))
+}
+
+func serveAsset(w http.ResponseWriter, r *http.Request, asset string) {
+	data, err := Asset(asset)
+
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	http.ServeContent(w, r, r.URL.Path, time.Time{}, bytes.NewReader(data))
+}

web/index.html

@@ -0,0 +1,266 @@
+<!doctype html>
+<html lang="en">
+<head>
+	<meta charset="utf-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1.0">
+	<meta name="description" content="Ironsmith - A simple, script driven continuous integration tool">
+
+	<title>Ironsmith - A simple, script driven continuous integration tool</title>
+
+	<link rel="stylesheet" href="/css/pure-min.css">
+
+	<style>
+		.container {
+			padding-right: 15px;
+			padding-left: 15px;
+			margin-right: auto;
+			margin-left: auto;
+		}
+
+		@media (min-width: 768px) {
+			.container {
+				padding-right: 40px;
+				padding-left: 40px;
+			}
+		}
+
+		.center-block {
+			display: block;
+			margin-left: auto;
+			margin-right: auto;
+		}
+
+		.text-center {
+			text-align: center;
+		}
+
+		/*tables*/
+
+		.table-responsive {
+			margin-left: auto;
+			margin-right: auto;
+			overflow-x: auto;
+		}
+
+		.table-responsive table {
+			width: 100%;
+			max-width: 100%;
+		}
+
+		/* error */
+		.error {
+			display: inline-block;
+			background-color: red;
+			color: white;
+			border-radius: 4px;
+			padding: .5em 1em;
+			margin: 10px;
+		}
+
+		/* breadcrumbs */
+
+		#breadcrumbs {
+			margin-bottom: 10px;
+		}
+
+		.breadcrumb-separator {
+			color: #ccc;
+			font-weight: bold;
+			font-size: 2em;
+		}
+
+
+		.pull-left {
+			float: left;
+		}
+		.pull-right {
+			float: right;
+		}
+	
+		.timestamp {
+			font-size: .75em;
+			color: #777;
+			margin-left: 10px;
+		}
+
+		.log {
+			margin-left: 15px;
+			margin-right: 15px;
+		}
+
+		.log > pre {
+			margin-left: 15px;
+		}
+	</style>
+</head>
+<body>
+<script id="tMain" type="text/ractive">
+<div class="container pure-g">
+	<div class="pure-u-1">
+		<h3 class="text-center">Iron Smith</h3>
+		{{#if error}}
+			<div class="text-center">
+				<span class="error">{{error}}</span>
+			</div>
+		{{/if}}
+		<div id="breadcrumbs" class="pure-menu pure-menu-horizontal text-center">
+			<ul class="pure-menu-list">
+				<li class="pure-menu-item">
+					<a href="/" class="pure-menu-link">Project List</a>
+				</li>
+				{{#if project}}
+					<li class="pure-menu-item">
+						<span class="breadcrumb-separator">/</span>
+					</li>
+					{{#if !version && !currentStage}}
+						<li class="pure-menu-item pure-menu-has-children" decorator="menu">
+							<a href="#" id="projectMenu" class="pure-menu-link">{{project.name}}</a>
+							<ul class="pure-menu-children">
+								<li class="pure-menu-item">
+									<a href="#" class="pure-menu-link" on-click="triggerBuild">Trigger Build</a>
+								</li>
+							</ul>
+						</li>
+					{{else}}
+						<li class="pure-menu-item">
+							<a href="/project/{{project.id}}" class="pure-menu-link">{{project.name}}</a>
+						</li>
+					{{/if}}
+				{{/if}}
+				{{#if project && version}}
+					<li class="pure-menu-item">
+						<span class="breadcrumb-separator">/</span>
+					</li>
+					<li class="pure-menu-item">
+						<a href="/project/{{project.id}}/{{version}}" class="pure-menu-link">{{version}}</a>
+					</li>
+				{{/if}}
+				{{#if project && version && currentStage}}
+					<li class="pure-menu-item">
+						<span class="breadcrumb-separator">/</span>
+					</li>
+					<li class="pure-menu-item">
+						<a href="/project/{{project.id}}/{{version}}/{{currentStage}}" class="pure-menu-link">{{currentStage}}</a>
+					</li>
+				{{/if}}
+			</ul>
+		</div>
+		{{#if !project}}
+			{{>projects}}
+		{{elseif !version}}
+			{{>project}}
+		{{else}}
+			{{>version}}
+		{{/if}}
+	</div>
+</div>
+
+
+{{#partial projects}}
+<div class="table-responsive">
+<table class="pure-table pure-table-striped">
+	<thead>
+		<tr>
+			<th>Project</th>
+			<th>Status</th>
+			<th>Last Version</th>
+			<th>Last Log</th>
+			<th>Last Release</th>
+			<th>Last Release File</th>
+		</tr>
+	</thead>
+	<tbody>
+		{{#projects:i}}	
+			<tr title="{{formatDate(.lastLog.when)}}">
+				<td><a href="/project/{{.id}}/">{{.name}}</a></td>
+				<td>{{.status}}</td>
+				<td>
+					<a href="/project/{{.id}}/{{.lastLog.version}}">{{.lastLog.version}}</a>
+				</td>
+				<td title="{{.lastLog.log}}">{{#if .lastLog.log && .lastLog.log.length > 150}}{{.lastLog.log.substring(0,150)}}...{{else}}{{.lastLog.log}}{{/if}}</td>
+				<td>
+					<a href="/project/{{.id}}/{{.releaseVersion}}">{{.releaseVersion}}</a>
+				</td>
+				<td>
+					{{#if releases[.id]}}
+						<a href="/release/{{.id}}?file">{{releases[id].fileName}}</a>	
+					{{else}}
+						No release file available
+					{{/if}}
+				</td>
+			</tr>
+		{{/projects}}
+	</tbody>
+</table>
+</div>
+{{/partial}}
+
+{{#partial project}}
+<div class="table-responsive">
+<table class="pure-table pure-table-striped">
+	<thead>
+		<tr>
+			<th>Version</th>
+			<th>Stage</th>
+			<th>Last Log</th>
+			<th>Release File</th>
+		</tr>
+	</thead>
+	<tbody>
+		{{#project.versions:i}}	
+			<tr title="{{formatDate(.when)}}">
+				<td>
+					<a href="/project/{{project.id}}/{{.version}}">{{.version}}</a>
+				</td>
+				<td>{{.stage}}</td>
+				<td title="{{.log}}">{{#if .log && .log.length > 150}}{{.log.substring(0,150)}}...{{else}}{{.log}}{{/if}}</td>
+				<td>
+					{{#if releases[project.id + .version]}}
+						<a href="/release/{{project.id}}/{{.version}}?file">{{releases[project.id + .version].fileName}}</a>	
+					{{/if}}
+				</td>
+			</tr>
+		{{/versions}}
+	</tbody>
+</table>
+</div>
+
+{{/partial}}
+
+{{#partial version}}
+<hr>
+{{#if releases[project.id + .version]}}
+	<a href="/release/{{project.id}}/{{.version}}?file" class="pull-right pure-button pure-button-primary">Download Release</a>
+{{/if}}
+
+<div class="pure-menu pure-menu-horizontal">
+	<ul class="pure-menu-list">
+		<li class="pure-menu-item {{#if !currentStage}}pure-menu-selected{{/if}}">
+			<a href="/project/{{project.id}}/{{.version}}/" class="pure-menu-link">All</a>
+		</li>
+		{{#stages:i}}
+		<li class="pure-menu-item {{#if currentStage && currentStage == .stage}}pure-menu-selected{{/if}}">
+			<a href="/project/{{project.id}}/{{version}}/{{.stage}}" class="pure-menu-link">{{.stage}}</a>
+		</li>
+		{{/stages}}
+	</ul>
+</div>
+<hr>
+<div class="log">
+	{{#if currentStage}}
+		<h3>{{currentStage}}<small class="timestamp">{{formatDate(logs.when)}}</small></h3>
+		<pre><samp>{{logs.log}}</samp></pre>
+	{{else}}
+		{{#stages:i}}
+			<h3>{{.stage}}<small class="timestamp">{{formatDate(.when)}}</small></h3>
+			<pre><samp>{{.log}}</samp></pre>
+		{{/stages}}
+	{{/if}}
+</div>
+{{/partial}}
+
+</script>
+<script src="/js/ractive.min.js"></script>
+<script src="/js/index.js"></script>
+</body>
+</html>

web/js/index.js

@@ -0,0 +1,425 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+/* jshint  strict: true */
+
+Ractive.DEBUG = false;
+
+(function() {
+    "use strict";
+
+    var r = new Ractive({
+        el: "body",
+        template: "#tMain",
+        data: function() {
+            return {
+                project: null,
+                version: null,
+                stages: null,
+                currentStage: null,
+                logs: null,
+                projects: [],
+                error: null,
+                formatDate: formatDate,
+                releases: {},
+            };
+        },
+        decorators: {
+            menu: function(node) {
+                new PureDropdown(node);
+                return {
+                    teardown: function() {
+                        return;
+                    },
+                };
+            },
+        },
+    });
+
+    setPaths();
+
+
+    r.on({
+        "triggerBuild": function(event) {
+            event.original.preventDefault();
+            var secret = window.prompt("Please enter the trigger secret for this project:");
+            triggerBuild(r.get("project.id"), secret);
+        },
+    });
+
+
+    function triggerBuild(projectID, secret) {
+        ajax("POST", "/trigger/" + projectID, {
+                secret: secret
+            },
+            function(result) {
+                window.location = "/";
+            },
+            function(result) {
+                r.set("error", err(result).message);
+            });
+    }
+
+
+    function setPaths() {
+        var paths = window.location.pathname.split("/");
+
+        if (paths.length <= 1) {
+            getProjects();
+            return;
+        }
+        if (!paths[1]) {
+            getProjects();
+            return;
+        }
+
+        if (paths[1] == "project") {
+            if (paths[2]) {
+                if (paths[3]) {
+                    if (paths[4]) {
+                        getStage(paths[2], paths[3], paths[4]);
+                    }
+                    getVersion(paths[2], paths[3]);
+                }
+                getProject(paths[2]);
+            }
+            getProjects();
+            return;
+        }
+
+        r.set("error", "Path Not found!");
+    }
+
+
+    function getProjects() {
+        get("/log/",
+            function(result) {
+                for (var i = 0; i < result.data.length; i++) {
+                    setStatus(result.data[i]);
+                    hasRelease(result.data[i].id, "");
+                }
+
+                result.data.sort(function(a, b) {
+                    if (a.name > b.name) {
+                        return 1;
+                    }
+                    if (a.name < b.name) {
+                        return -1;
+                    }
+                    return 0;
+                });
+                r.set("projects", result.data);
+
+                window.setTimeout(getProjects, 10000);
+            },
+            function(result) {
+                r.set("error", err(result).message);
+            });
+    }
+
+    function getProject(id) {
+        get("/log/" + id,
+            function(result) {
+                r.set("project", result.data);
+                if (result.data.versions) {
+                    for (var i = 0; i < result.data.versions.length; i++) {
+                        hasRelease(result.data.id, result.data.versions[i].version);
+                    }
+                }
+            },
+            function(result) {
+                r.set("error", err(result).message);
+            });
+    }
+
+    function getVersion(id, version) {
+        get("/log/" + id + "/" + version,
+            function(result) {
+                if (!result.data || !result.data.length || !result.data[0].version) {
+                    r.set("version", version);
+                } else {
+                    r.set("version", result.data[0].version);
+                }
+                r.set("stages", result.data);
+            },
+            function(result) {
+                r.set("error", err(result).message);
+            });
+    }
+
+    function getStage(id, version, stage) {
+        get("/log/" + id + "/" + version + "/" + stage,
+            function(result) {
+                r.set("logs", result.data);
+                r.set("currentStage", stage);
+            },
+            function(result) {
+                r.set("error", err(result).message);
+            });
+    }
+
+    function hasRelease(id, version) {
+        /*/release/<project-id>/<version>*/
+        get("/release/" + id + "/" + version,
+            function(result) {
+                r.set("releases." + id + version, result.data);
+            },
+            function(result) {
+                r.set("releases." + id + version, undefined);
+            });
+
+
+    }
+
+    function setStatus(project) {
+        //statuses 
+        if (project.stage != "waiting") {
+            project.status = project.stage;
+        } else if (!project.lastLog || !project.lastLog.version) {
+            project.status = "waiting";
+        } else if (project.lastLog.version.trim() == project.releaseVersion.trim()) {
+            project.status = "Successfully Released";
+        } else {
+            if (project.lastLog.stage == "loading") {
+                project.status = "Load Failing";
+            } else if (project.lastLog.stage == "fetching") {
+                project.status = "Fetch Failing";
+            } else if (project.lastLog.stage == "building") {
+                project.status = "Build Failing";
+            } else if (project.lastLog.stage == "testing") {
+                project.status = "Tests Failing";
+            } else if (project.lastLog.stage == "releasing") {
+                project.status = "Release Failing";
+            } else {
+                project.status = "Failing";
+            }
+        }
+    }
+
+})();
+
+function ajax(type, url, data, success, error) {
+    "use strict";
+    var req = new XMLHttpRequest();
+    req.open(type, url);
+
+    if (success || error) {
+        req.onload = function() {
+            if (req.status >= 200 && req.status < 400) {
+                if (success && typeof success === 'function') {
+                    var result;
+                    try {
+                        result = JSON.parse(req.responseText);
+                    } catch (e) {
+                        result = "";
+                    }
+                    success(result);
+                }
+                return;
+            }
+
+            //failed
+            if (error && typeof error === 'function') {
+                error(req);
+            }
+        };
+        req.onerror = function() {
+            if (error && typeof error === 'function') {
+                error(req);
+            }
+        };
+    }
+
+    var sendData;
+    if (type != "get") {
+        req.setRequestHeader("Content-Type", "application/json");
+        sendData = JSON.stringify(data);
+    }
+
+    req.send(sendData);
+}
+
+function get(url, success, error) {
+    "use strict";
+    ajax("GET", url, null, success, error);
+}
+
+function err(response) {
+    "use strict";
+    var error = {
+        message: "An error occurred",
+    };
+
+    if (typeof response === "string") {
+        error.message = response;
+    } else {
+        error.message = JSON.parse(response.responseText).message;
+    }
+
+    return error;
+}
+
+function formatDate(strDate) {
+    "use strict";
+    var date = new Date(strDate);
+    if (!date) {
+        return "";
+    }
+    return date.toLocaleDateString() + " at " + date.toLocaleTimeString();
+}
+
+
+function PureDropdown(dropdownParent) {
+    "use strict";
+
+    var PREFIX = 'pure-',
+        ACTIVE_CLASS_NAME = PREFIX + 'menu-active',
+        ARIA_ROLE = 'role',
+        ARIA_HIDDEN = 'aria-hidden',
+        MENU_OPEN = 0,
+        MENU_CLOSED = 1,
+        MENU_PARENT_CLASS_NAME = 'pure-menu-has-children',
+        MENU_ACTIVE_SELECTOR = '.pure-menu-active',
+        MENU_LINK_SELECTOR = '.pure-menu-link',
+        MENU_SELECTOR = '.pure-menu-children',
+        DISMISS_EVENT = (window.hasOwnProperty &&
+            window.hasOwnProperty('ontouchstart')) ?
+        'touchstart' : 'mousedown',
+
+        ARROW_KEYS_ENABLED = true,
+
+        ddm = this; // drop down menu
+
+    this._state = MENU_CLOSED;
+
+    this.show = function() {
+        if (this._state !== MENU_OPEN) {
+            this._dropdownParent.classList.add(ACTIVE_CLASS_NAME);
+            this._menu.setAttribute(ARIA_HIDDEN, false);
+            this._state = MENU_OPEN;
+        }
+    };
+
+    this.hide = function() {
+        if (this._state !== MENU_CLOSED) {
+            this._dropdownParent.classList.remove(ACTIVE_CLASS_NAME);
+            this._menu.setAttribute(ARIA_HIDDEN, true);
+            this._link.focus();
+            this._state = MENU_CLOSED;
+        }
+    };
+
+    this.toggle = function() {
+        this[this._state === MENU_CLOSED ? 'show' : 'hide']();
+    };
+
+    this.halt = function(e) {
+        e.stopPropagation();
+        e.preventDefault();
+    };
+
+    this._dropdownParent = dropdownParent;
+    this._link = this._dropdownParent.querySelector(MENU_LINK_SELECTOR);
+    this._menu = this._dropdownParent.querySelector(MENU_SELECTOR);
+    this._firstMenuLink = this._menu.querySelector(MENU_LINK_SELECTOR);
+
+    // Set ARIA attributes
+    this._link.setAttribute('aria-haspopup', 'true');
+    this._menu.setAttribute(ARIA_ROLE, 'menu');
+    this._menu.setAttribute('aria-labelledby', this._link.getAttribute('id'));
+    this._menu.setAttribute('aria-hidden', 'true');
+    [].forEach.call(
+        this._menu.querySelectorAll('li'),
+        function(el) {
+            el.setAttribute(ARIA_ROLE, 'presentation');
+        }
+    );
+    [].forEach.call(
+        this._menu.querySelectorAll('a'),
+        function(el) {
+            el.setAttribute(ARIA_ROLE, 'menuitem');
+        }
+    );
+
+    // Toggle on click
+    this._link.addEventListener('click', function(e) {
+        e.stopPropagation();
+        e.preventDefault();
+        ddm.toggle();
+    });
+
+    // Keyboard navigation
+    document.addEventListener('keydown', function(e) {
+        var currentLink,
+            previousSibling,
+            nextSibling,
+            previousLink,
+            nextLink;
+
+        // if the menu isn't active, ignore
+        if (ddm._state !== MENU_OPEN) {
+            return;
+        }
+
+        // if the menu is the parent of an open, active submenu, ignore
+        if (ddm._menu.querySelector(MENU_ACTIVE_SELECTOR)) {
+            return;
+        }
+
+        currentLink = ddm._menu.querySelector(':focus');
+
+        // Dismiss an open menu on ESC
+        if (e.keyCode === 27) {
+            /* Esc */
+            ddm.halt(e);
+            ddm.hide();
+        }
+        // Go to the next link on down arrow
+        else if (ARROW_KEYS_ENABLED && e.keyCode === 40) {
+            /* Down arrow */
+            ddm.halt(e);
+            // get the nextSibling (an LI) of the current link's LI
+            nextSibling = (currentLink) ? currentLink.parentNode.nextSibling : null;
+            // if the nextSibling is a text node (not an element), go to the next one
+            while (nextSibling && nextSibling.nodeType !== 1) {
+                nextSibling = nextSibling.nextSibling;
+            }
+            nextLink = (nextSibling) ? nextSibling.querySelector('.pure-menu-link') : null;
+            // if there is no currently focused link, focus the first one
+            if (!currentLink) {
+                ddm._menu.querySelector('.pure-menu-link').focus();
+            } else if (nextLink) {
+                nextLink.focus();
+            }
+        }
+        // Go to the previous link on up arrow
+        else if (ARROW_KEYS_ENABLED && e.keyCode === 38) {
+            /* Up arrow */
+            ddm.halt(e);
+            // get the currently focused link
+            previousSibling = (currentLink) ? currentLink.parentNode.previousSibling : null;
+            while (previousSibling && previousSibling.nodeType !== 1) {
+                previousSibling = previousSibling.previousSibling;
+            }
+            previousLink = (previousSibling) ? previousSibling.querySelector('.pure-menu-link') : null;
+            // if there is no currently focused link, focus the last link
+            if (!currentLink) {
+                ddm._menu.querySelector('.pure-menu-item:last-child .pure-menu-link').focus();
+            }
+            // else if there is a previous item, go to the previous item
+            else if (previousLink) {
+                previousLink.focus();
+            }
+        }
+    });
+
+    // Dismiss an open menu on outside event
+    document.addEventListener(DISMISS_EVENT, function(e) {
+        var target = e.target;
+        if (target !== ddm._link && !ddm._menu.contains(target)) {
+            ddm.hide();
+            ddm._link.blur();
+        }
+    });
+
+}

webProject.go

@@ -0,0 +1,264 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"git.townsourced.com/townsourced/ironsmith/datastore"
+)
+
+// /path/<project-id>/<version>/<stage>
+func splitPath(path string) (project, version, stage string) {
+	s := strings.Split(path, "/")
+	if len(s) < 3 {
+		return
+	}
+
+	project, _ = url.QueryUnescape(s[2])
+	if len(s) < 4 {
+		return
+	}
+
+	version, _ = url.QueryUnescape(s[3])
+
+	if len(s) < 5 {
+		return
+	}
+	stage, _ = url.QueryUnescape(s[4])
+
+	return
+}
+
+/*
+	/log/ - list all projects
+	/log/<project-id> - list all versions in a project,  POST triggers new builds
+	/log/<project-id>/<version> - list combined output of all stages for a given version
+	/log/<project-id>/<version>/<stage> - list output of a given stage of a given version
+*/
+func logGet(w http.ResponseWriter, r *http.Request) {
+	prj, ver, stg := splitPath(r.URL.Path)
+
+	if prj == "" {
+		///log/ - list all projects
+		pList, err := projects.webList()
+		if errHandled(err, w, r) {
+			return
+		}
+
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   pList,
+		})
+
+		return
+	}
+
+	project, ok := projects.get(prj)
+	if !ok {
+		four04(w, r)
+		return
+	}
+
+	//project found
+
+	if ver == "" {
+		///log/<project-id> - list all versions in a project
+
+		vers, err := project.versions()
+		if errHandled(err, w, r) {
+			return
+		}
+
+		prjData, err := project.webData()
+		if errHandled(err, w, r) {
+			return
+		}
+
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data: struct {
+				*webProject
+				Versions []*datastore.Log `json:"versions"`
+			}{
+				webProject: prjData,
+				Versions:   vers,
+			},
+		})
+		return
+	}
+
+	//ver found
+	if stg == "" {
+		///log/<project-id>/<version> - list combined output of all stages for a given version
+		logs, err := project.versionLog(ver)
+		if errHandled(err, w, r) {
+			return
+		}
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   logs,
+		})
+		return
+	}
+
+	//stage found
+	///log/<project-id>/<version>/<stage> - list output of a given stage of a given version
+
+	log, err := project.stageLog(ver, stg)
+	if errHandled(err, w, r) {
+		return
+	}
+
+	respondJsend(w, &JSend{
+		Status: statusSuccess,
+		Data:   log,
+	})
+	return
+}
+
+/*
+	/release/<project-id>/<version>
+
+	/release/<project-id> - list last release for a given project
+		?all returns all the releases for a project ?file returns the last release file
+	/release/<project-id>/<version> - list release for a given project version ?file returns the file for a given release version
+*/
+func releaseGet(w http.ResponseWriter, r *http.Request) {
+	prj, ver, _ := splitPath(r.URL.Path)
+
+	values := r.URL.Query()
+
+	_, all := values["all"]
+	_, file := values["file"]
+
+	if prj == "" {
+		four04(w, r)
+		return
+	}
+
+	project, ok := projects.get(prj)
+	if !ok {
+		four04(w, r)
+		return
+	}
+
+	//project found
+
+	if ver == "" {
+		///release/<project-id> - list last release for a given project
+		//	?all returns all the releases for a project ?file returns the last release file
+
+		if all {
+			releases, err := project.releases()
+			if errHandled(err, w, r) {
+				return
+			}
+
+			respondJsend(w, &JSend{
+				Status: statusSuccess,
+				Data:   releases,
+			})
+			return
+
+		}
+
+		last, err := project.lastRelease()
+		if errHandled(err, w, r) {
+			return
+		}
+
+		if file {
+			fileData, err := project.releaseFile(last.FileKey)
+			if errHandled(err, w, r) {
+				return
+			}
+
+			w.Header().Add("Content-disposition", `attachment; filename="`+last.FileName+`"`)
+			http.ServeContent(w, r, last.FileName, time.Time{}, bytes.NewReader(fileData))
+			return
+		}
+
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   last,
+		})
+
+		return
+	}
+
+	//ver found
+	// /release/<project-id>/<version> - list release for a given project version ?file returns the file for a given release version
+
+	release, err := project.releaseData(ver)
+	if errHandled(err, w, r) {
+		return
+	}
+
+	if file {
+		fileData, err := project.releaseFile(release.FileKey)
+		if errHandled(err, w, r) {
+			return
+		}
+
+		w.Header().Add("Content-disposition", `attachment; filename="`+release.FileName+`"`)
+		http.ServeContent(w, r, release.FileName, time.Time{}, bytes.NewReader(fileData))
+		return
+	}
+
+	respondJsend(w, &JSend{
+		Status: statusSuccess,
+		Data:   release,
+	})
+}
+
+type triggerInput struct {
+	Secret string `json:"secret"`
+}
+
+/*trigger routes
+/trigger/<project-id>
+	Triggers a project to start a cycle
+*/
+func triggerPost(w http.ResponseWriter, r *http.Request) {
+	prj, _, _ := splitPath(r.URL.Path)
+
+	if prj == "" {
+		four04(w, r)
+		return
+	}
+
+	project, ok := projects.get(prj)
+	if !ok {
+		four04(w, r)
+		return
+	}
+
+	if strings.TrimSpace(project.TriggerSecret) == "" {
+		four04(w, r)
+		return
+	}
+
+	input := &triggerInput{}
+	if errHandled(parseInput(r, input), w, r) {
+		return
+	}
+
+	if input.Secret != project.TriggerSecret {
+		errHandled(&Fail{
+			Message:    "Invalid trigger secret for this project",
+			HTTPStatus: http.StatusUnauthorized,
+		}, w, r)
+		return
+	}
+
+	go func() {
+		project.load(true)
+	}()
+}