Finished backend work

Added last of the web REST endpoints.
Added processing lock to ensure that only one project cycle is running
at a time.

Basically all that's left is the web front end

bindata.go

@@ -0,0 +1,261 @@
+// Code generated by go-bindata.
+// sources:
+// web/css/pure-min.css
+// web/index.html
+// web/js/index.js
+// web/js/ractive.min.js
+// DO NOT EDIT!
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// bindataRead reads the given file from disk. It returns an error on failure.
+func bindataRead(path, name string) ([]byte, error) {
+	buf, err := ioutil.ReadFile(path)
+	if err != nil {
+		err = fmt.Errorf("Error reading asset %s at %s: %v", name, path, err)
+	}
+	return buf, err
+}
+
+type asset struct {
+	bytes []byte
+	info  os.FileInfo
+}
+
+// webCssPureMinCss reads file data from disk. It returns an error on failure.
+func webCssPureMinCss() (*asset, error) {
+	path := "/home/tshannon/workspace/go/src/git.townsourced.com/ironsmith/web/css/pure-min.css"
+	name := "web/css/pure-min.css"
+	bytes, err := bindataRead(path, name)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
+	}
+
+	a := &asset{bytes: bytes, info: fi}
+	return a, err
+}
+
+// webIndexHtml reads file data from disk. It returns an error on failure.
+func webIndexHtml() (*asset, error) {
+	path := "/home/tshannon/workspace/go/src/git.townsourced.com/ironsmith/web/index.html"
+	name := "web/index.html"
+	bytes, err := bindataRead(path, name)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
+	}
+
+	a := &asset{bytes: bytes, info: fi}
+	return a, err
+}
+
+// webJsIndexJs reads file data from disk. It returns an error on failure.
+func webJsIndexJs() (*asset, error) {
+	path := "/home/tshannon/workspace/go/src/git.townsourced.com/ironsmith/web/js/index.js"
+	name := "web/js/index.js"
+	bytes, err := bindataRead(path, name)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
+	}
+
+	a := &asset{bytes: bytes, info: fi}
+	return a, err
+}
+
+// webJsRactiveMinJs reads file data from disk. It returns an error on failure.
+func webJsRactiveMinJs() (*asset, error) {
+	path := "/home/tshannon/workspace/go/src/git.townsourced.com/ironsmith/web/js/ractive.min.js"
+	name := "web/js/ractive.min.js"
+	bytes, err := bindataRead(path, name)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		err = fmt.Errorf("Error reading asset info %s at %s: %v", name, path, err)
+	}
+
+	a := &asset{bytes: bytes, info: fi}
+	return a, err
+}
+
+// Asset loads and returns the asset for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+func Asset(name string) ([]byte, error) {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[cannonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
+		}
+		return a.bytes, nil
+	}
+	return nil, fmt.Errorf("Asset %s not found", name)
+}
+
+// MustAsset is like Asset but panics when Asset would return an error.
+// It simplifies safe initialization of global variables.
+func MustAsset(name string) []byte {
+	a, err := Asset(name)
+	if err != nil {
+		panic("asset: Asset(" + name + "): " + err.Error())
+	}
+
+	return a
+}
+
+// AssetInfo loads and returns the asset info for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+func AssetInfo(name string) (os.FileInfo, error) {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[cannonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
+		}
+		return a.info, nil
+	}
+	return nil, fmt.Errorf("AssetInfo %s not found", name)
+}
+
+// AssetNames returns the names of the assets.
+func AssetNames() []string {
+	names := make([]string, 0, len(_bindata))
+	for name := range _bindata {
+		names = append(names, name)
+	}
+	return names
+}
+
+// _bindata is a table, holding each asset generator, mapped to its name.
+var _bindata = map[string]func() (*asset, error){
+	"web/css/pure-min.css": webCssPureMinCss,
+	"web/index.html": webIndexHtml,
+	"web/js/index.js": webJsIndexJs,
+	"web/js/ractive.min.js": webJsRactiveMinJs,
+}
+
+// AssetDir returns the file names below a certain
+// directory embedded in the file by go-bindata.
+// For example if you run go-bindata on data/... and data contains the
+// following hierarchy:
+//     data/
+//       foo.txt
+//       img/
+//         a.png
+//         b.png
+// then AssetDir("data") would return []string{"foo.txt", "img"}
+// AssetDir("data/img") would return []string{"a.png", "b.png"}
+// AssetDir("foo.txt") and AssetDir("notexist") would return an error
+// AssetDir("") will return []string{"data"}.
+func AssetDir(name string) ([]string, error) {
+	node := _bintree
+	if len(name) != 0 {
+		cannonicalName := strings.Replace(name, "\\", "/", -1)
+		pathList := strings.Split(cannonicalName, "/")
+		for _, p := range pathList {
+			node = node.Children[p]
+			if node == nil {
+				return nil, fmt.Errorf("Asset %s not found", name)
+			}
+		}
+	}
+	if node.Func != nil {
+		return nil, fmt.Errorf("Asset %s not found", name)
+	}
+	rv := make([]string, 0, len(node.Children))
+	for childName := range node.Children {
+		rv = append(rv, childName)
+	}
+	return rv, nil
+}
+
+type bintree struct {
+	Func     func() (*asset, error)
+	Children map[string]*bintree
+}
+var _bintree = &bintree{nil, map[string]*bintree{
+	"web": &bintree{nil, map[string]*bintree{
+		"css": &bintree{nil, map[string]*bintree{
+			"pure-min.css": &bintree{webCssPureMinCss, map[string]*bintree{}},
+		}},
+		"index.html": &bintree{webIndexHtml, map[string]*bintree{}},
+		"js": &bintree{nil, map[string]*bintree{
+			"index.js": &bintree{webJsIndexJs, map[string]*bintree{}},
+			"ractive.min.js": &bintree{webJsRactiveMinJs, map[string]*bintree{}},
+		}},
+	}},
+}}
+
+// RestoreAsset restores an asset under the given directory
+func RestoreAsset(dir, name string) error {
+	data, err := Asset(name)
+	if err != nil {
+		return err
+	}
+	info, err := AssetInfo(name)
+	if err != nil {
+		return err
+	}
+	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
+	if err != nil {
+		return err
+	}
+	err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// RestoreAssets restores an asset under the given directory recursively
+func RestoreAssets(dir, name string) error {
+	children, err := AssetDir(name)
+	// File
+	if err != nil {
+		return RestoreAsset(dir, name)
+	}
+	// Dir
+	for _, child := range children {
+		err = RestoreAssets(dir, filepath.Join(name, child))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func _filePath(dir, name string) string {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
+}
+

cycle.go

@@ -8,87 +8,15 @@ import (
 	"encoding/json"
 	"errors"
 	"io/ioutil"
-	"log"
 	"os"
-	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
 	"git.townsourced.com/ironsmith/datastore"
 )
 
-func (p *Project) errHandled(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	if p.ds == nil {
-		log.Printf("Error in project %s: %s\n", p.id(), err)
-		return true
-	}
-	defer func() {
-		err = p.ds.Close()
-		if err != nil {
-			log.Printf("Error closing the datastore for project %s: %s\n", p.id(), err)
-		}
-		p.ds = nil
-
-		//clean up version folder if it exists
-
-		if p.version != "" {
-			err = os.RemoveAll(p.verDir())
-			log.Printf("Error deleting the version directory project %s version %s: %s\n",
-				p.id(), p.version, err)
-
-		}
-	}()
-
-	lerr := p.ds.AddLog(p.version, p.stage, err.Error())
-	if lerr != nil {
-		log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s",
-			p.id(), err, lerr)
-	}
-
-	return true
-}
-
-func (p *Project) id() string {
-	if p.filename == "" {
-		panic("invalid project filename")
-	}
-	return strings.TrimSuffix(p.filename, filepath.Ext(p.filename))
-}
-
-func (p *Project) dir() string {
-	return filepath.Join(dataDir, p.id())
-}
-
-func (p *Project) verDir() string {
-	return filepath.Join(p.dir(), p.version)
-}
-
-// prepData makes sure the project's data folder and data store is created
-/*
-	folder structure
-	projectDataFolder/<project-name>/<project-version>
-
-*/
-func (p *Project) prepData() error {
-	err := os.MkdirAll(p.dir(), 0777)
-	if err != nil {
-		return err
-	}
-
-	p.ds, err = datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith"))
-
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 /*
 Project life cycle:
 (Load Project file) -> (Fetch) -> (Build) -> (Test) -> (Release) - > (Sleep for polling period) ->
@@ -99,18 +27,26 @@ Project life cycle:
 // load is the beginning of the cycle.  Loads / reloads the project file to make sure that the scripts are up-to-date
 // call's fetch and triggers the next poll if one exists
 func (p *Project) load() {
-	p.version = ""
+	p.processing.Lock() // ensure only one cycle is running at a time per project
+	defer p.processing.Unlock()
+
+	p.setStage(stageLoad)
+	p.setVersion("")
 
 	if p.filename == "" {
 		p.errHandled(errors.New("Invalid project file name"))
 		return
 	}
 
-	if !projects.exists(p.filename) {
+	if _, ok := projects.get(p.id()); !ok {
 		// project has been deleted
 		// don't continue polling
-		// move project data to deleted folder
-		p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, p.id())))
+		// move project data to deleted folder with a timestamp
+		if p.errHandled(p.close()) {
+			return
+		}
+		p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir,
+			strconv.FormatInt(time.Now().Unix(), 10), p.id())))
 		return
 	}
 
@@ -119,102 +55,108 @@ func (p *Project) load() {
 		return
 	}
 
-	if p.errHandled(json.Unmarshal(data, p)) {
+	new := &Project{}
+	if p.errHandled(json.Unmarshal(data, new)) {
 		return
 	}
 
-	p.stage = stageLoad
-
-	if p.errHandled(p.prepData()) {
-		return
-	}
-
-	if p.PollInterval != "" {
-		p.poll, err = time.ParseDuration(p.PollInterval)
-		if p.errHandled(err) {
-			p.poll = 0
-		}
-	}
+	p.setData(new)
 
 	p.fetch()
 
+	p.setStage(stageWait)
+
 	//full cycle completed
 
 	if p.poll > 0 {
 		//start polling
-		time.AfterFunc(p.poll, p.load)
+		go func() {
+			time.AfterFunc(p.poll, p.load)
+		}()
 	}
 }
 
-// fetch first runs the version script and checks the returned version against the latest version in the
-// project database. If the version hasn't changed, then it breaks out of the cycle early doing nothing
-// if the version has changed, then it runs the fetch script
+// fetch first runs the fetch script into a temporary directory
+// then it runs the version script in the temp directory to see if there is a newer version of the
+// fetched code, if there is then the temp dir is renamed to the version name
 func (p *Project) fetch() {
-	p.stage = stageFetch
-	verCmd := &exec.Cmd{
-		Path: p.Version,
-		Dir:  p.dir(),
-	}
+	p.setStage(stageFetch)
 
-	version, err := verCmd.Output()
-
-	if p.errHandled(err) {
+	if p.Fetch == "" {
 		return
 	}
 
-	p.version = string(version)
+	tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10))
 
-	lVer, err := p.ds.LatestVersion()
-	if err != datastore.ErrNotFound && p.errHandled(err) {
-		return
-	}
-
-	if p.version == lVer {
-		// no new build
-		return
-	}
-
-	if p.errHandled(os.MkdirAll(p.verDir(), 0777)) {
+	if p.errHandled(os.MkdirAll(tempDir, 0777)) {
 		return
 	}
 
 	//fetch project
-	fetchCmd := &exec.Cmd{
-		Path: p.Fetch,
-		Dir:  p.verDir(),
-	}
-
-	fetchResult, err := fetchCmd.Output()
+	fetchResult, err := runCmd(p.Fetch, tempDir)
 	if p.errHandled(err) {
 		return
 	}
 
-	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(fetchResult))) {
+	// fetched succesfully, determine version
+	version, err := runCmd(p.Version, tempDir)
+
+	if p.errHandled(err) {
 		return
 	}
 
-	// fetched succesfully, onto the build stage
-	p.build()
+	p.setVersion(strings.TrimSpace(string(version)))
 
+	lVer, err := p.ds.LastVersion("")
+	if err != datastore.ErrNotFound && p.errHandled(err) {
+		return
+	}
+
+	if p.version == "" || p.version == lVer {
+		// no new build clean up temp dir
+		p.errHandled(os.RemoveAll(tempDir))
+
+		vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
+		return
+	}
+
+	//remove any existing data that matches version hash
+	if p.errHandled(os.RemoveAll(p.workingDir())) {
+		return
+	}
+
+	//new version move tempdir to workingDir
+	if p.errHandled(os.Rename(tempDir, p.workingDir())) {
+		// cleanup temp dir if rename failed
+		p.errHandled(os.RemoveAll(tempDir))
+		return
+	}
+
+	//log fetch results
+	if p.errHandled(p.ds.AddLog(p.version, p.stage, string(fetchResult))) {
+		return
+	}
+
+	// continue to build
+	p.build()
 }
 
 // build  runs the build scripts to build the project which should result in the a single file
 // configured in the ReleaseFile section of the project file
 func (p *Project) build() {
-	p.stage = stageBuild
+	p.setStage(stageBuild)
 
-	buildCmd := &exec.Cmd{
-		Path: p.Build,
-		Dir:  p.verDir(),
+	if p.Build == "" {
+		return
 	}
 
-	output, err := buildCmd.Output()
+	output, err := runCmd(p.Build, p.workingDir())
 
 	if p.errHandled(err) {
 		return
 	}
 
-	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(output))) {
+	if p.errHandled(p.ds.AddLog(p.version, p.stage, string(output))) {
 		return
 	}
 
@@ -224,49 +166,45 @@ func (p *Project) build() {
 
 // test runs the test scripts
 func (p *Project) test() {
-	p.stage = stageTest
+	p.setStage(stageTest)
 
-	testCmd := &exec.Cmd{
-		Path: p.Test,
-		Dir:  p.verDir(),
+	if p.Test == "" {
+		return
 	}
-
-	output, err := testCmd.Output()
+	output, err := runCmd(p.Test, p.workingDir())
 
 	if p.errHandled(err) {
 		return
 	}
 
-	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(output))) {
+	if p.errHandled(p.ds.AddLog(p.version, p.stage, string(output))) {
 		return
 	}
 
 	//  Tests passed, onto release
 	p.release()
-
 }
 
 // release runs the release scripts and builds the release file
 func (p *Project) release() {
-	p.stage = stageRelease
+	p.setStage(stageRelease)
 
-	releaseCmd := &exec.Cmd{
-		Path: p.Release,
-		Dir:  p.verDir(),
+	if p.Release == "" {
+		return
 	}
 
-	output, err := releaseCmd.Output()
+	output, err := runCmd(p.Release, p.workingDir())
 
 	if p.errHandled(err) {
 		return
 	}
 
-	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(output))) {
+	if p.errHandled(p.ds.AddLog(p.version, p.stage, string(output))) {
 		return
 	}
 
 	//get release file
-	f, err := os.Open(filepath.Join(p.verDir(), p.ReleaseFile))
+	f, err := os.Open(filepath.Join(p.workingDir(), p.ReleaseFile))
 	if p.errHandled(err) {
 		return
 	}
@@ -276,8 +214,12 @@ func (p *Project) release() {
 		return
 	}
 
-	if p.errHandled(p.ds.AddRelease(p.version, buff)) {
+	if p.errHandled(p.ds.AddRelease(p.version, p.ReleaseFile, buff)) {
 		return
 	}
 
+	//build successfull, remove working dir
+	p.errHandled(os.RemoveAll(p.workingDir()))
+
+	vlog("Project: %s Version %s built, tested, and released successfully.\n", p.id(), p.version)
 }

datastore/ds.go

@@ -15,8 +15,10 @@ import (
 	"github.com/boltdb/bolt"
 )
 
+//TODO: Move this all over to GobStore if I ever get around to finishing it
+
 // ErrNotFound is the error returned when a value cannot be found in the store for the given key
-var ErrNotFound = errors.New("Value not found")
+var ErrNotFound = errors.New("Value not found in datastore")
 
 // Store is a datastore for getting and setting data for a given ironsmith project
 // run on top of a Bolt DB file
@@ -64,12 +66,12 @@ func Open(filename string) (*Store, error) {
 
 // Close closes the bolt datastore
 func (ds *Store) Close() error {
-	return ds.Close()
+	return ds.bolt.Close()
 }
 
-func (ds *Store) get(bucket string, key TimeKey, result interface{}) error {
+func (ds *Store) get(bucket string, key []byte, result interface{}) error {
 	return ds.bolt.View(func(tx *bolt.Tx) error {
-		dsValue := tx.Bucket([]byte(bucket)).Get(key.Bytes())
+		dsValue := tx.Bucket([]byte(bucket)).Get(key)
 
 		if dsValue == nil {
 			return ErrNotFound
@@ -88,7 +90,7 @@ func (ds *Store) get(bucket string, key TimeKey, result interface{}) error {
 	})
 }
 
-func (ds *Store) put(bucket string, key TimeKey, value interface{}) error {
+func (ds *Store) put(bucket string, key []byte, value interface{}) error {
 	var err error
 	dsValue, ok := value.([]byte)
 	if !ok {
@@ -99,12 +101,12 @@ func (ds *Store) put(bucket string, key TimeKey, value interface{}) error {
 	}
 
 	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		return tx.Bucket([]byte(bucket)).Put(key.Bytes(), dsValue)
+		return tx.Bucket([]byte(bucket)).Put(key, dsValue)
 	})
 }
 
-func (ds *Store) delete(bucket string, key TimeKey) error {
+func (ds *Store) delete(bucket string, key []byte) error {
 	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		return tx.Bucket([]byte(bucket)).Delete(key.Bytes())
+		return tx.Bucket([]byte(bucket)).Delete(key)
 	})
 }

datastore/key.go

@@ -29,10 +29,6 @@ func NewTimeKey() TimeKey {
 	nsec := t.Nanosecond()
 
 	return TimeKey{
-		rBits[0], //random
-		rBits[1],
-		rBits[2],
-		rBits[3],
 		byte(sec >> 56), // seconds
 		byte(sec >> 48),
 		byte(sec >> 40),
@@ -45,12 +41,16 @@ func NewTimeKey() TimeKey {
 		byte(nsec >> 16),
 		byte(nsec >> 8),
 		byte(nsec),
+		rBits[0], //random
+		rBits[1],
+		rBits[2],
+		rBits[3],
 	}
 }
 
 // Time returns the time portion of a timekey
 func (k TimeKey) Time() time.Time {
-	buf := k[4:]
+	buf := k[:]
 
 	sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 |
 		int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56
@@ -83,3 +83,44 @@ func (k TimeKey) UUID() string {
 func (k TimeKey) Bytes() []byte {
 	return []byte(k[:])
 }
+
+// String returns the string representation of a timekey
+func (k TimeKey) String() string {
+	return k.UUID()
+}
+
+// MarshalJSON implements JSON marshaler
+func (k *TimeKey) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + k.String() + `"`), nil
+}
+
+// UnmarshalJSON implements JSON unmarshaler
+func (k *TimeKey) UnmarshalJSON(buf []byte) error {
+	// drop quotes
+	buf = buf[1 : len(buf)-1]
+	_, err := hex.Decode(k[0:4], buf[0:8])
+	if err != nil {
+		return err
+	}
+
+	_, err = hex.Decode(k[4:6], buf[9:13])
+	if err != nil {
+		return err
+	}
+
+	_, err = hex.Decode(k[6:8], buf[14:18])
+	if err != nil {
+		return err
+	}
+
+	_, err = hex.Decode(k[8:10], buf[19:23])
+	if err != nil {
+		return err
+	}
+	_, err = hex.Decode(k[10:], buf[24:])
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

datastore/log.go

@@ -11,11 +11,12 @@ import (
 	"github.com/boltdb/bolt"
 )
 
-type log struct {
-	When    time.Time `json:"when"`
-	Version string    `json:"version"`
-	Stage   string    `json:"stage"`
-	Log     string    `json:"log"`
+// Log is a version log entry for a project
+type Log struct {
+	When    time.Time `json:"when,omitempty"`
+	Version string    `json:"version,omitempty"`
+	Stage   string    `json:"stage,omitempty"`
+	Log     string    `json:"log,omitempty"`
 }
 
 const bucketLog = "log"
@@ -24,41 +25,156 @@ const bucketLog = "log"
 func (ds *Store) AddLog(version, stage, entry string) error {
 	key := NewTimeKey()
 
-	data := &log{
+	data := &Log{
 		When:    key.Time(),
 		Version: version,
 		Stage:   stage,
 		Log:     entry,
 	}
 
-	return ds.put(bucketLog, key, data)
+	return ds.put(bucketLog, key.Bytes(), data)
 }
 
-// LatestVersion returns the latest version for the current project
-func (ds *Store) LatestVersion() (string, error) {
+// LastVersion returns the last version in the log for the given stage.  If stage is blank,
+// then it returns the last of any stage
+func (ds *Store) LastVersion(stage string) (string, error) {
 	version := ""
 
 	err := ds.bolt.View(func(tx *bolt.Tx) error {
 		c := tx.Bucket([]byte(bucketLog)).Cursor()
 
 		for k, v := c.Last(); k != nil; k, v = c.Prev() {
-			l := &log{}
+			l := &Log{}
 			err := json.Unmarshal(v, l)
 			if err != nil {
 				return err
 			}
 
 			if l.Version != "" {
-				version = l.Version
+				if stage == "" || l.Stage == stage {
+					version = l.Version
+					return nil
+				}
+			}
+		}
+
+		return nil // not found return blank
+	})
+
+	if err != nil {
+		return "", err
+	}
+
+	return version, nil
+}
+
+// Versions lists the versions in a given project, including the last stage that version got to
+func (ds *Store) Versions() ([]*Log, error) {
+	var vers []*Log
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		var current = ""
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			l := &Log{}
+			err := json.Unmarshal(v, l)
+			if err != nil {
+				return err
+			}
+
+			// capture the newest entry for each version
+			if l.Version != current {
+				l.Log = "" // only care about date, ver and stage
+				vers = append(vers, l)
+				current = l.Version
+			}
+
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return vers, nil
+}
+
+// VersionLog returns all the log entries for a given version
+func (ds *Store) VersionLog(version string) ([]*Log, error) {
+	var logs []*Log
+
+	if version == "" {
+		return logs, nil
+	}
+
+	verFound := false
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			l := &Log{}
+			err := json.Unmarshal(v, l)
+			if err != nil {
+				return err
+			}
+
+			if verFound && l.Version != version {
 				return nil
 			}
+
+			if l.Version == version {
+				logs = append(logs, l)
+				verFound = true
+			}
+
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return logs, nil
+}
+
+// StageLog returns the log entry for a given version + stage
+func (ds *Store) StageLog(version, stage string) (*Log, error) {
+	var entry *Log
+
+	if version == "" || stage == "" {
+		return nil, ErrNotFound
+	}
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			l := &Log{}
+			err := json.Unmarshal(v, l)
+			if err != nil {
+				return err
+			}
+
+			if l.Version == version && l.Stage == stage {
+				entry = l
+				return nil
+			}
+
 		}
 
 		return ErrNotFound
 	})
 
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	return version, nil
+
+	return entry, nil
 }

datastore/releases.go

@@ -11,10 +11,12 @@ import (
 	"github.com/boltdb/bolt"
 )
 
-type release struct {
-	When    time.Time `json:"when"`
-	Version string    `json:"version"`
-	FileKey TimeKey   `json:"file"`
+// Release is a record of the fully built and ready to deploy release file
+type Release struct {
+	When     time.Time `json:"when"`
+	Version  string    `json:"version"`
+	FileName string    `json:"fileName"`
+	FileKey  TimeKey   `json:"fileKey"`
 }
 
 const (
@@ -23,14 +25,14 @@ const (
 )
 
 // AddRelease adds a new Release
-func (ds *Store) AddRelease(version string, fileData []byte) error {
-	key := NewTimeKey()
+func (ds *Store) AddRelease(version, fileName string, fileData []byte) error {
 	fileKey := NewTimeKey()
 
-	r := &release{
-		When:    key.Time(),
-		Version: version,
-		FileKey: fileKey,
+	r := &Release{
+		When:     fileKey.Time(),
+		Version:  version,
+		FileName: fileName,
+		FileKey:  fileKey,
 	}
 
 	dsValue, err := json.Marshal(r)
@@ -39,7 +41,7 @@ func (ds *Store) AddRelease(version string, fileData []byte) error {
 	}
 
 	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		err = tx.Bucket([]byte(bucketReleases)).Put(key.Bytes(), dsValue)
+		err = tx.Bucket([]byte(bucketReleases)).Put([]byte(version), dsValue)
 		if err != nil {
 			return err
 		}
@@ -47,3 +49,78 @@ func (ds *Store) AddRelease(version string, fileData []byte) error {
 		return tx.Bucket([]byte(bucketFiles)).Put(fileKey.Bytes(), fileData)
 	})
 }
+
+// ReleaseFile returns a specific file from a release for the given file key
+func (ds *Store) ReleaseFile(fileKey TimeKey) ([]byte, error) {
+	var fileData []byte
+	err := ds.get(bucketFiles, fileKey.Bytes(), fileData)
+	if err != nil {
+		return nil, err
+	}
+
+	return fileData, nil
+}
+
+// Release gets the release record for a specific version
+func (ds *Store) Release(version string) (*Release, error) {
+	r := &Release{}
+	err := ds.get(bucketReleases, []byte(version), r)
+	if err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// Releases lists all the releases in a given project
+func (ds *Store) Releases() ([]*Release, error) {
+	var vers []*Release
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketReleases)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			r := &Release{}
+			err := json.Unmarshal(v, r)
+			if err != nil {
+				return err
+			}
+
+			vers = append(vers, r)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return vers, nil
+}
+
+// LastRelease lists the last release for a project
+func (ds *Store) LastRelease() (*Release, error) {
+	r := &Release{}
+
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketReleases)).Cursor()
+
+		_, v := c.Last()
+		if v == nil {
+			return ErrNotFound
+		}
+
+		err := json.Unmarshal(v, r)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return r, nil
+}

error.go

@@ -0,0 +1,147 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+
+	"git.townsourced.com/ironsmith/datastore"
+)
+
+const (
+	acceptHTML = "text/html"
+)
+
+// Err404 is a standard 404 error response
+var Err404 = errors.New("Resource not found")
+
+func errHandled(err error, w http.ResponseWriter, r *http.Request) bool {
+	if err == nil {
+		return false
+	}
+
+	if err == datastore.ErrNotFound {
+		four04(w, r)
+		return true
+	}
+
+	var status, errMsg string
+
+	errMsg = err.Error()
+
+	switch err.(type) {
+
+	case *Fail:
+		status = statusFail
+	case *http.ProtocolError, *json.SyntaxError, *json.UnmarshalTypeError:
+		//Hardcoded external errors which can bubble up to the end users
+		// without exposing internal server information, make them failures
+		err = FailFromErr(err)
+		status = statusFail
+
+		errMsg = fmt.Sprintf("We had trouble parsing your input, please check your input and try again: %s", err)
+	default:
+		status = statusError
+		log.Printf("An error has occurred from a web request: %s", errMsg)
+		errMsg = "An internal server error has occurred"
+	}
+
+	if status == statusFail {
+		respondJsendCode(w, &JSend{
+			Status:  status,
+			Message: errMsg,
+			Data:    err.(*Fail).Data,
+		}, err.(*Fail).HTTPStatus)
+	} else {
+		respondJsend(w, &JSend{
+			Status:  status,
+			Message: errMsg,
+		})
+	}
+
+	return true
+}
+
+// four04 is a standard 404 response if request header accepts text/html
+// they'll get a 404 page, otherwise a json response
+func four04(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Content-Type", "application/json")
+
+	response := &JSend{
+		Status:  statusFail,
+		Message: "Resource not found",
+		Data:    r.URL.String(),
+	}
+
+	w.WriteHeader(http.StatusNotFound)
+
+	result, err := json.Marshal(response)
+	if err != nil {
+		log.Printf("Error marshalling 404 response: %s", err)
+		return
+	}
+
+	_, err = w.Write(result)
+	if err != nil {
+		log.Printf("Error in four04: %s", err)
+	}
+}
+
+// Fail is an error whose contents can be exposed to the client and is usually the result
+// of incorrect client input
+type Fail struct {
+	Message    string      `json:"message,omitempty"`
+	Data       interface{} `json:"data,omitempty"`
+	HTTPStatus int         `json:"-"` //gets set in the error response
+}
+
+func (f *Fail) Error() string {
+	return f.Message
+}
+
+// NewFail creates a new failure, data is optional
+func NewFail(message string, data ...interface{}) error {
+	return &Fail{
+		Message:    message,
+		Data:       data,
+		HTTPStatus: 0,
+	}
+}
+
+// FailFromErr returns a new failure based on the passed in error, data is optional
+// if passed in error is nil, then nil is returned
+func FailFromErr(err error, data ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+	return NewFail(err.Error(), data...)
+}
+
+// IsEqual tests whether an error is equal to another error / failure
+func (f *Fail) IsEqual(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	return err.Error() == f.Error()
+}
+
+// IsFail tests whether the passed in error is a failure
+func IsFail(err error) bool {
+	if err == nil {
+		return false
+	}
+	switch err.(type) {
+	case *Fail:
+		return true
+	default:
+		return false
+	}
+}

exec.go

@@ -0,0 +1,33 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+func runCmd(cmd, dir string) ([]byte, error) {
+	s := strings.Fields(cmd)
+
+	var args []string
+
+	if len(s) > 1 {
+		args = s[1:]
+	}
+
+	ec := exec.Command(s[0], args...)
+
+	ec.Dir = dir
+
+	vlog("Executing command: %s in dir %s\n", cmd, dir)
+
+	result, err := ec.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("%s:\n%s", err, result)
+	}
+	return result, nil
+}

json.go

@@ -0,0 +1,100 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+)
+
+const (
+	statusSuccess = "success"
+	statusError   = "error"
+	statusFail    = "fail"
+)
+
+const maxJSONSize = 1 << 20 //10MB
+
+var errInputTooLarge = &Fail{
+	Message:    "Input size is too large, please check your input and try again",
+	HTTPStatus: http.StatusRequestEntityTooLarge,
+}
+
+// JSend is the standard format for a response from townsourced
+type JSend struct {
+	Status   string      `json:"status"`
+	Data     interface{} `json:"data,omitempty"`
+	Message  string      `json:"message,omitempty"`
+	Failures []error     `json:"failures,omitempty"`
+	More     bool        `json:"more,omitempty"` // more data exists for this request
+}
+
+//respondJsend marshalls the input into a json byte array
+// and writes it to the reponse with appropriate header
+func respondJsend(w http.ResponseWriter, response *JSend) {
+	respondJsendCode(w, response, 0)
+}
+
+// respondJsendCode is the same as respondJSend, but lets you specify a status code
+func respondJsendCode(w http.ResponseWriter, response *JSend, statusCode int) {
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Content-Type", "application/json")
+
+	if len(response.Failures) > 0 && response.Message == "" {
+		response.Message = "One or more item has failed. Check the individual failures for details."
+	}
+
+	result, err := json.MarshalIndent(response, "", "    ")
+	if err != nil {
+		log.Printf("Error marshalling response: %s", err)
+
+		result, _ = json.Marshal(&JSend{
+			Status:  statusError,
+			Message: "An internal error occurred, and we'll look into it.",
+		})
+	}
+
+	if statusCode <= 0 {
+		switch response.Status {
+		case statusFail:
+			w.WriteHeader(http.StatusBadRequest)
+		case statusError:
+			w.WriteHeader(http.StatusInternalServerError)
+		}
+		//default is status 200
+	} else {
+		w.WriteHeader(statusCode)
+	}
+
+	_, err = w.Write(result)
+	if err != nil {
+		log.Printf("Error writing jsend response: %s", err)
+	}
+}
+
+func parseInput(r *http.Request, result interface{}) error {
+	lr := &io.LimitedReader{R: r.Body, N: maxJSONSize + 1}
+	buff, err := ioutil.ReadAll(lr)
+	if err != nil {
+		return err
+	}
+
+	if lr.N == 0 {
+		return errInputTooLarge
+	}
+
+	if len(buff) == 0 {
+		return nil
+	}
+
+	err = json.Unmarshal(buff, result)
+	if err != nil {
+		return err
+	}
+	return nil
+}

log.go

@@ -0,0 +1,13 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import "log"
+
+func vlog(format string, v ...interface{}) {
+	if verbose {
+		log.Printf(format, v...)
+	}
+}

main.go

@@ -5,8 +5,10 @@
 package main
 
 import (
+	"flag"
 	"log"
 	"os"
+	"os/signal"
 	"path/filepath"
 
 	"git.townsourced.com/config"
@@ -16,23 +18,46 @@ import (
 var (
 	projectDir = "./projects" // /etc/ironsmith/
 	dataDir    = "./data"     // /var/ironsmith/
-	address    = "http://localhost:8026"
+	address    = ":8026"
 	certFile   = ""
 	keyFile    = ""
 )
 
+//flags
+var (
+	verbose = false
+)
+
+func init() {
+	flag.BoolVar(&verbose, "v", false, "Verbose prints to stdOut every command and stage as it processes")
+
+	//Capture program shutdown, to make sure everything shuts down nicely
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		for sig := range c {
+			if sig == os.Interrupt {
+				projects.stopAll()
+				os.Exit(0)
+			}
+		}
+	}()
+}
+
 func main() {
+	flag.Parse()
+
 	settingPaths := config.StandardFileLocations("ironsmith/settings.json")
-	log.Println("IronSmith will use settings files in the following locations (in order of priority):")
+	vlog("IronSmith will use settings files in the following locations (in order of priority):\n")
 	for i := range settingPaths {
-		log.Println("\t", settingPaths[i])
+		vlog("\t%s\n", settingPaths[i])
 	}
 	cfg, err := config.LoadOrCreate(settingPaths...)
 	if err != nil {
 		log.Fatalf("Error loading or creating IronSmith settings file: %s", err)
 	}
 
-	log.Printf("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
+	vlog("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
 
 	projectDir = cfg.String("projectDir", projectDir)
 	dataDir = cfg.String("dataDir", dataDir)
@@ -40,6 +65,9 @@ func main() {
 	certFile = cfg.String("certFile", certFile)
 	keyFile = cfg.String("keyFile", keyFile)
 
+	vlog("Project Definition Directory: %s\n", projectDir)
+	vlog("Project Data Directory: %s\n", dataDir)
+
 	//prep dirs
 	err = os.MkdirAll(filepath.Join(projectDir, enabledProjectDir), 0777)
 	if err != nil {
@@ -61,5 +89,11 @@ func main() {
 	if err != nil {
 		log.Fatalf("Error loading projects: %s", err)
 	}
-	//start server
+
+	//start web server
+	err = startServer()
+	if err != nil {
+		log.Fatalf("Error Starting web server: %s", err)
+	}
+
 }

project.go

@@ -5,10 +5,13 @@
 package main
 
 import (
+	"crypto/sha1"
 	"encoding/json"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
@@ -22,11 +25,12 @@ const (
 
 //stages
 const (
-	stageLoad    = "load"
-	stageFetch   = "fetch"
-	stageBuild   = "build"
-	stageTest    = "test"
-	stageRelease = "release"
+	stageLoad    = "loading"
+	stageFetch   = "fetching"
+	stageBuild   = "building"
+	stageTest    = "testing"
+	stageRelease = "releasing"
+	stageWait    = "waiting"
 )
 
 const projectFilePoll = 30 * time.Second
@@ -51,14 +55,256 @@ type Project struct {
 	Version string `json:"version"` //Script to generate the version num of the current build, should be indempotent
 
 	ReleaseFile   string `json:"releaseFile"`
-	PollInterval  string `json:"pollInterval"`  // if not poll interval is specified, this project is trigger only
-	TriggerSecret string `json:"triggerSecret"` //secret to be included with a trigger call
+	PollInterval  string `json:"pollInterval,omitempty"`  // if not poll interval is specified, this project is trigger only
+	TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call
 
 	filename string
 	poll     time.Duration
 	ds       *datastore.Store
 	stage    string
+	status   string
 	version  string
+	hash     string
+
+	sync.RWMutex
+	processing sync.Mutex
+}
+
+func (p *Project) errHandled(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	vlog("Error in project %s: %s\n", p.id(), err)
+
+	if p.ds == nil {
+		log.Printf("Error in project %s: %s\n", p.id(), err)
+		return true
+	}
+	defer func() {
+		//clean up version folder if it exists
+		if p.version != "" {
+			err = os.RemoveAll(p.workingDir())
+			if err != nil {
+				log.Printf("Error deleting the version directory project %s version %s: %s\n",
+					p.id(), p.version, err)
+			}
+
+		}
+	}()
+
+	lerr := p.ds.AddLog(p.version, p.stage, err.Error())
+	if lerr != nil {
+		log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s",
+			p.id(), err, lerr)
+	}
+
+	return true
+}
+
+func projectID(filename string) string {
+	return strings.TrimSuffix(filename, filepath.Ext(filename))
+}
+
+func (p *Project) id() string {
+	if p.filename == "" {
+		panic("invalid project filename")
+	}
+	return projectID(p.filename)
+}
+
+func (p *Project) dir() string {
+	return filepath.Join(dataDir, p.id())
+}
+
+func (p *Project) workingDir() string {
+	if p.hash == "" {
+		panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
+	}
+
+	//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
+	// timestamp on instead would work just fine, but I like having the working dir tied directly to the
+	// version returned by project script
+
+	return filepath.Join(p.dir(), p.hash)
+}
+
+// prepData makes sure the project's data folder and data store is created
+/*
+	folder structure
+	projectDataFolder/<project-name>/<project-version>
+
+*/
+func (p *Project) open() error {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.ds != nil {
+		return nil
+	}
+
+	err := os.MkdirAll(p.dir(), 0777)
+	if err != nil {
+		return err
+	}
+
+	ds, err := datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith"))
+	if err != nil {
+		return err
+	}
+
+	p.ds = ds
+
+	return nil
+}
+
+func (p *Project) setVersion(version string) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.version = version
+	if version == "" {
+		p.hash = ""
+		return
+	}
+
+	p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(version)))
+}
+
+func (p *Project) setStage(stage string) {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.version != "" {
+		vlog("Entering %s stage for Project: %s Version: %s\n", stage, p.id(), p.version)
+	} else {
+		vlog("Entering %s stage for Project: %s\n", stage, p.id())
+	}
+
+	p.stage = stage
+}
+
+type webProject struct {
+	ID             string `json:"id"`
+	Name           string `json:"name"`
+	ReleaseVersion string `json:"releaseVersion"` //last successfully released version
+	LastVersion    string `json:"lastVersion"`    //last version success or otherwise
+	Stage          string `json:"stage"`          // current stage
+}
+
+func (p *Project) webData() (*webProject, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	last, err := p.ds.LastVersion("")
+	if err != nil {
+		return nil, err
+	}
+
+	release, err := p.ds.LastVersion(stageRelease)
+	if err != nil {
+		return nil, err
+	}
+
+	d := &webProject{
+		Name:           p.Name,
+		ID:             p.id(),
+		LastVersion:    last,
+		ReleaseVersion: release,
+		Stage:          p.stage,
+	}
+
+	return d, nil
+}
+
+func (p *Project) versions() ([]*datastore.Log, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.Versions()
+}
+
+func (p *Project) versionLog(version string) ([]*datastore.Log, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.VersionLog(version)
+}
+
+func (p *Project) stageLog(version, stage string) (*datastore.Log, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.StageLog(version, stage)
+}
+
+func (p *Project) releases() ([]*datastore.Release, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.Releases()
+}
+
+func (p *Project) lastRelease() (*datastore.Release, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.LastRelease()
+}
+func (p *Project) releaseData(version string) (*datastore.Release, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.Release(version)
+}
+
+func (p *Project) releaseFile(fileKey datastore.TimeKey) ([]byte, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	return p.ds.ReleaseFile(fileKey)
+}
+
+// releaseFile
+
+func (p *Project) setData(new *Project) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.Name = new.Name
+
+	p.Fetch = new.Fetch
+	p.Build = new.Build
+	p.Test = new.Test
+	p.Release = new.Release
+	p.Version = new.Version
+
+	p.ReleaseFile = new.ReleaseFile
+	p.PollInterval = new.PollInterval
+	p.TriggerSecret = new.TriggerSecret
+
+	if p.PollInterval != "" {
+		var err error
+		p.poll, err = time.ParseDuration(p.PollInterval)
+		if p.errHandled(err) {
+			p.poll = 0
+		}
+	}
+}
+
+func (p *Project) close() error {
+	p.Lock()
+	defer p.Unlock()
+	if p.ds == nil {
+		return nil
+	}
+	err := p.ds.Close()
+	if err != nil {
+		return err
+	}
+
+	p.ds = nil
+	return nil
 }
 
 const projectTemplateFilename = "template.project.json"
@@ -66,12 +312,12 @@ const projectTemplateFilename = "template.project.json"
 var projectTemplate = &Project{
 	Name:    "Template Project",
 	Fetch:   "git clone root@git.townsourced.com:tshannon/ironsmith.git .",
-	Build:   "sh ./ironsmith/build.sh",
-	Test:    "sh ./ironsmith/test.sh",
-	Release: "sh ./ironsmith/release.sh",
+	Build:   "go build -a -v -o ironsmith",
+	Test:    "go test ./...",
+	Release: "tar -czf release.tar.gz ironsmith",
 	Version: "git describe --tags --long",
 
-	ReleaseFile:  `json:"./ironsmith/release.tar.gz"`,
+	ReleaseFile:  "release.tar.gz",
 	PollInterval: "15m",
 }
 
@@ -79,6 +325,7 @@ func prepTemplateProject() error {
 	filename := filepath.Join(projectDir, projectTemplateFilename)
 	_, err := os.Stat(filename)
 	if os.IsNotExist(err) {
+		vlog("Creating template project file in %s", filename)
 		f, err := os.Create(filename)
 		defer func() {
 			if cerr := f.Close(); cerr != nil && err == nil {
@@ -117,6 +364,7 @@ var projects = projectList{
 }
 
 func (p *projectList) load() error {
+	vlog("Loading projects from the enabled definitions in %s\n", filepath.Join(projectDir, enabledProjectDir))
 	dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
 	defer func() {
 		if cerr := dir.Close(); cerr != nil && err == nil {
@@ -144,15 +392,16 @@ func (p *projectList) load() error {
 	return nil
 }
 
-func (p *projectList) exists(name string) bool {
+func (p *projectList) get(name string) (*Project, bool) {
 	p.RLock()
 	defer p.RUnlock()
 
-	_, ok := p.data[name]
-	return ok
+	prj, ok := p.data[name]
+	return prj, ok
 }
 
 func (p *projectList) add(name string) {
+	vlog("Adding project %s to the project list.\n", name)
 	p.Lock()
 	defer p.Unlock()
 
@@ -161,9 +410,14 @@ func (p *projectList) add(name string) {
 		Name:     name,
 		stage:    stageLoad,
 	}
-	p.data[name] = prj
+	p.data[projectID(name)] = prj
 
 	go func() {
+		err := prj.open()
+		if err != nil {
+			log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err)
+			return
+		}
 		prj.load()
 	}()
 }
@@ -176,16 +430,48 @@ func (p *projectList) removeMissing(names []string) {
 	for i := range p.data {
 		found := false
 		for k := range names {
-			if names[k] == i {
+			if projectID(names[k]) == i {
 				found = true
 			}
 		}
 		if !found {
+			vlog("Removing project %s from the project list, because the project file was removed.\n",
+				i)
 			delete(p.data, i)
 		}
 	}
 }
 
+func (p *projectList) stopAll() {
+	p.RLock()
+	defer p.RUnlock()
+
+	for i := range p.data {
+		err := p.data[i].close()
+		if err != nil {
+			log.Printf("Error closing project datastore for Project: %s Error: %s\n", p.data[i].id(), err)
+		}
+	}
+}
+
+func (p *projectList) webList() ([]*webProject, error) {
+	p.RLock()
+	defer p.RUnlock()
+
+	list := make([]*webProject, 0, len(p.data))
+
+	for i := range p.data {
+		prj, err := p.data[i].webData()
+		if err != nil {
+			return nil, err
+		}
+
+		list = append(list, prj)
+	}
+
+	return list, nil
+}
+
 // startProjectLoader polls for new projects
 func startProjectLoader() {
 	dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
@@ -196,13 +482,13 @@ func startProjectLoader() {
 	}()
 
 	if err != nil {
-		log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", dir, err)
+		log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", projectDir, err)
 		return
 	}
 
 	files, err := dir.Readdir(0)
 	if err != nil {
-		log.Printf("Error in startProjectLoader reading the dir %s: %s\n", dir, err)
+		log.Printf("Error in startProjectLoader reading the dir %s: %s\n", projectDir, err)
 		return
 	}
 
@@ -211,7 +497,7 @@ func startProjectLoader() {
 	for i := range files {
 		if !files[i].IsDir() && filepath.Ext(files[i].Name()) == ".json" {
 			names[i] = files[i].Name()
-			if !projects.exists(files[i].Name()) {
+			if _, ok := projects.get(projectID(files[i].Name())); !ok {
 				projects.add(files[i].Name())
 			}
 		}

server.go

@@ -0,0 +1,139 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"net/http"
+	"path"
+	"time"
+)
+
+var webRoot *http.ServeMux
+
+func startServer() error {
+	var err error
+
+	routes()
+
+	server := &http.Server{
+		Handler: webRoot,
+		Addr:    address,
+	}
+
+	if certFile == "" || keyFile == "" {
+		err = server.ListenAndServe()
+	} else {
+		server.Addr = address
+		err = server.ListenAndServeTLS(certFile, keyFile)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type methodHandler struct {
+	get    http.HandlerFunc
+	post   http.HandlerFunc
+	put    http.HandlerFunc
+	delete http.HandlerFunc
+}
+
+func (m *methodHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if m.get == nil {
+		m.get = http.NotFound
+	}
+	if m.post == nil {
+		m.post = http.NotFound
+	}
+	if m.put == nil {
+		m.put = http.NotFound
+	}
+	if m.delete == nil {
+		m.delete = http.NotFound
+	}
+	switch r.Method {
+	case "GET":
+		m.get(w, r)
+		return
+	case "POST":
+		m.post(w, r)
+		return
+	case "PUT":
+		m.put(w, r)
+		return
+	case "DELETE":
+		m.delete(w, r)
+		return
+	default:
+		http.NotFound(w, r)
+		return
+	}
+}
+
+/*
+log Routes
+	/log/<project-id>/<version>/<stage>
+
+	/log/ - list all projects
+	/log/<project-id> - list all versions in a project, triggers new builds
+	/log/<project-id>/<version> - list combined output of all stages for a given version
+	/log/<project-id>/<version>/<stage> - list output of a given stage of a given version
+
+release routes
+	/release/<project-id>/<version>
+
+	/release/<project-id> - list last release for a given project  ?all returns all the releases for a project
+	/release/<project-id>/<version> - list release for a given project version
+
+trigger routes
+	/trigger/<project-id>
+		Triggers a project to start a cycle
+*/
+
+func routes() {
+	webRoot = http.NewServeMux()
+
+	webRoot.Handle("/", &methodHandler{
+		get: rootGet,
+	})
+
+	webRoot.Handle("/log/", &methodHandler{
+		get: logGet,
+	})
+
+	webRoot.Handle("/release/", &methodHandler{
+		get: releaseGet,
+	})
+
+	webRoot.Handle("/trigger/", &methodHandler{
+		post: triggerPost,
+	})
+
+}
+
+func rootGet(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path == "/" {
+		//send index.html
+		serveAsset(w, r, "web/index.html")
+		return
+	}
+
+	serveAsset(w, r, path.Join("web", r.URL.Path))
+}
+
+func serveAsset(w http.ResponseWriter, r *http.Request, asset string) {
+	data, err := Asset(asset)
+
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	http.ServeContent(w, r, r.URL.Path, time.Time{}, bytes.NewReader(data))
+}

web/index.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html lang="en">
+<head>
+	<meta charset="utf-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1.0">
+	<meta name="description" content="Ironsmith - A simple, script driven continuous integration tool">
+
+	<title>Ironsmith - A simple, script driven continuous integration tool</title>
+
+	<link rel="stylesheet" href="/css/pure-min.css">
+
+	<style>
+
+	</style>
+</head>
+<body>
+<script id="tMain" type="text/ractive">
+	
+</script>
+<script src="/js/ractive.min.js"></script>
+<script src="/js/index.js"></script>
+</body>
+</html>

web/js/index.js

@@ -0,0 +1,19 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+/* jshint  strict: true */
+
+(function() {
+    "use strict";
+
+	 var r = new Ractive({
+        el: "body",
+        template: "#tMain",
+		data: function() {
+			return {
+
+			};
+		},
+	});
+
+})();

webProject.go

@@ -0,0 +1,245 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// /path/<project-id>/<version>/<stage>
+func splitPath(path string) (project, version, stage string) {
+	s := strings.Split(path, "/")
+	if len(s) < 3 {
+		return
+	}
+
+	project, _ = url.QueryUnescape(s[2])
+	if len(s) < 4 {
+		return
+	}
+
+	version, _ = url.QueryUnescape(s[3])
+
+	if len(s) < 5 {
+		return
+	}
+	stage, _ = url.QueryUnescape(s[4])
+
+	return
+}
+
+/*
+	/log/ - list all projects
+	/log/<project-id> - list all versions in a project,  POST triggers new builds
+	/log/<project-id>/<version> - list combined output of all stages for a given version
+	/log/<project-id>/<version>/<stage> - list output of a given stage of a given version
+*/
+func logGet(w http.ResponseWriter, r *http.Request) {
+	prj, ver, stg := splitPath(r.URL.Path)
+
+	if prj == "" {
+		///log/ - list all projects
+		pList, err := projects.webList()
+		if errHandled(err, w, r) {
+			return
+		}
+
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   pList,
+		})
+
+		return
+	}
+
+	project, ok := projects.get(prj)
+	if !ok {
+		four04(w, r)
+		return
+	}
+
+	//project found
+
+	if ver == "" {
+		///log/<project-id> - list all versions in a project
+
+		vers, err := project.versions()
+		if errHandled(err, w, r) {
+			return
+		}
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   vers,
+		})
+		return
+	}
+
+	//ver found
+	if stg == "" {
+		///log/<project-id>/<version> - list combined output of all stages for a given version
+		logs, err := project.versionLog(ver)
+		if errHandled(err, w, r) {
+			return
+		}
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   logs,
+		})
+		return
+	}
+
+	//stage found
+	///log/<project-id>/<version>/<stage> - list output of a given stage of a given version
+
+	log, err := project.stageLog(ver, stg)
+	if errHandled(err, w, r) {
+		return
+	}
+
+	respondJsend(w, &JSend{
+		Status: statusSuccess,
+		Data:   log,
+	})
+	return
+}
+
+/*
+	/release/<project-id>/<version>
+
+	/release/<project-id> - list last release for a given project
+		?all returns all the releases for a project ?file returns the last release file
+	/release/<project-id>/<version> - list release for a given project version ?file returns the file for a given release version
+*/
+func releaseGet(w http.ResponseWriter, r *http.Request) {
+	prj, ver, _ := splitPath(r.URL.Path)
+
+	values := r.URL.Query()
+
+	_, all := values["all"]
+	_, file := values["file"]
+
+	if prj == "" {
+		four04(w, r)
+		return
+	}
+
+	project, ok := projects.get(prj)
+	if !ok {
+		four04(w, r)
+		return
+	}
+
+	//project found
+
+	if ver == "" {
+		///release/<project-id> - list last release for a given project
+		//	?all returns all the releases for a project ?file returns the last release file
+
+		if all {
+			releases, err := project.releases()
+			if errHandled(err, w, r) {
+				return
+			}
+
+			respondJsend(w, &JSend{
+				Status: statusSuccess,
+				Data:   releases,
+			})
+			return
+
+		}
+
+		last, err := project.lastRelease()
+		if errHandled(err, w, r) {
+			return
+		}
+
+		if file {
+			fileData, err := project.releaseFile(last.FileKey)
+			if errHandled(err, w, r) {
+				return
+			}
+
+			w.Header().Add("Content-disposition", `attachment; filename="`+last.FileName+`"`)
+			http.ServeContent(w, r, last.FileName, time.Time{}, bytes.NewReader(fileData))
+			return
+		}
+
+		respondJsend(w, &JSend{
+			Status: statusSuccess,
+			Data:   last,
+		})
+
+		return
+	}
+
+	//ver found
+	// /release/<project-id>/<version> - list release for a given project version ?file returns the file for a given release version
+
+	release, err := project.releaseData(ver)
+	if errHandled(err, w, r) {
+		return
+	}
+
+	if file {
+		fileData, err := project.releaseFile(release.FileKey)
+		if errHandled(err, w, r) {
+			return
+		}
+
+		w.Header().Add("Content-disposition", `attachment; filename="`+release.FileName+`"`)
+		http.ServeContent(w, r, release.FileName, time.Time{}, bytes.NewReader(fileData))
+		return
+	}
+
+	respondJsend(w, &JSend{
+		Status: statusSuccess,
+		Data:   release,
+	})
+}
+
+type triggerInput struct {
+	Secret string `json:"secret"`
+}
+
+/*trigger routes
+/trigger/<project-id>
+	Triggers a project to start a cycle
+*/
+func triggerPost(w http.ResponseWriter, r *http.Request) {
+	prj, _, _ := splitPath(r.URL.Path)
+
+	if prj == "" {
+		four04(w, r)
+		return
+	}
+
+	project, ok := projects.get(prj)
+	if !ok {
+		four04(w, r)
+		return
+	}
+
+	input := &triggerInput{}
+	if errHandled(parseInput(r, input), w, r) {
+		return
+	}
+
+	if input.Secret != project.TriggerSecret {
+		errHandled(&Fail{
+			Message:    "Invalid trigger secret for this project",
+			HTTPStatus: http.StatusUnauthorized,
+		}, w, r)
+		return
+	}
+
+	go func() {
+		project.load()
+	}()
+}