Added logging and proper execution of commands

Fixed several issues, and ran a basic test on ironsmith itself.

Need to prevent subsequent builds on same versions

cycle.go

@@ -5,13 +5,15 @@
 package main
 
 import (
+	"crypto/sha1"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
-	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -23,6 +25,8 @@ func (p *Project) errHandled(err error) bool {
 		return false
 	}
 
+	vlog("Error in project %s: %s\n", p.id(), err)
+
 	if p.ds == nil {
 		log.Printf("Error in project %s: %s\n", p.id(), err)
 		return true
@@ -37,9 +41,11 @@ func (p *Project) errHandled(err error) bool {
 		//clean up version folder if it exists
 
 		if p.version != "" {
-			err = os.RemoveAll(p.verDir())
+			err = os.RemoveAll(p.workingDir())
+			if err != nil {
 				log.Printf("Error deleting the version directory project %s version %s: %s\n",
 					p.id(), p.version, err)
+			}
 
 		}
 	}()
@@ -64,8 +70,16 @@ func (p *Project) dir() string {
 	return filepath.Join(dataDir, p.id())
 }
 
-func (p *Project) verDir() string {
+func (p *Project) workingDir() string {
-	return filepath.Join(p.dir(), p.version)
+	if p.hash == "" {
+		panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
+	}
+
+	//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
+	// timestamp on instead would work just fine, but I like having the working dir tied directly to the
+	// version returned by project script
+
+	return filepath.Join(p.dir(), p.hash)
 }
 
 // prepData makes sure the project's data folder and data store is created
@@ -100,6 +114,9 @@ Project life cycle:
 // call's fetch and triggers the next poll if one exists
 func (p *Project) load() {
 	p.version = ""
+	p.hash = ""
+
+	vlog("Entering %s stage for Project: %s\n", stageLoad, p.id())
 
 	if p.filename == "" {
 		p.errHandled(errors.New("Invalid project file name"))
@@ -109,8 +126,9 @@ func (p *Project) load() {
 	if !projects.exists(p.filename) {
 		// project has been deleted
 		// don't continue polling
-		// move project data to deleted folder
+		// move project data to deleted folder with a timestamp
-		p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, p.id())))
+		p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir,
+			strconv.FormatInt(time.Now().Unix(), 10), p.id())))
 		return
 	}
 
@@ -146,23 +164,33 @@ func (p *Project) load() {
 	}
 }
 
-// fetch first runs the version script and checks the returned version against the latest version in the
+// fetch first runs the fetch script into a temporary directory
-// project database. If the version hasn't changed, then it breaks out of the cycle early doing nothing
+// then it runs the version script in the temp directory to see if there is a newer version of the
-// if the version has changed, then it runs the fetch script
+// fetched code, if there is then the temp dir is renamed to the version name
 func (p *Project) fetch() {
 	p.stage = stageFetch
-	verCmd := &exec.Cmd{
+
-		Path: p.Version,
+	vlog("Entering %s stage for Project: %s\n", p.stage, p.id())
-		Dir:  p.dir(),
+	tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10))
+
+	if p.errHandled(os.MkdirAll(tempDir, 0777)) {
+		return
 	}
 
-	version, err := verCmd.Output()
+	//fetch project
+	fetchResult, err := runCmd(p.Fetch, tempDir)
+	if p.errHandled(err) {
+		return
+	}
+
+	// fetched succesfully, determine version
+	version, err := runCmd(p.Version, tempDir)
 
 	if p.errHandled(err) {
 		return
 	}
 
-	p.version = string(version)
+	p.version = strings.TrimSpace(string(version))
 
 	lVer, err := p.ds.LatestVersion()
 	if err != datastore.ErrNotFound && p.errHandled(err) {
@@ -170,30 +198,33 @@ func (p *Project) fetch() {
 	}
 
 	if p.version == lVer {
-		// no new build
+		// no new build clean up temp dir
+		p.errHandled(os.RemoveAll(tempDir))
+
+		vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
 		return
 	}
 
-	if p.errHandled(os.MkdirAll(p.verDir(), 0777)) {
+	p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(p.version)))
+
+	//remove any existing data that matches version hash
+	if p.errHandled(os.RemoveAll(p.workingDir())) {
 		return
 	}
 
-	//fetch project
+	//new version move tempdir to workingDir
-	fetchCmd := &exec.Cmd{
+	if p.errHandled(os.Rename(tempDir, p.workingDir())) {
-		Path: p.Fetch,
+		// cleanup temp dir if rename failed
-		Dir:  p.verDir(),
+		p.errHandled(os.RemoveAll(tempDir))
-	}
-
-	fetchResult, err := fetchCmd.Output()
-	if p.errHandled(err) {
 		return
 	}
 
+	//log fetch results
 	if p.errHandled(p.ds.AddLog(p.stage, p.version, string(fetchResult))) {
 		return
 	}
 
-	// fetched succesfully, onto the build stage
+	// continue to build
 	p.build()
 
 }
@@ -202,13 +233,9 @@ func (p *Project) fetch() {
 // configured in the ReleaseFile section of the project file
 func (p *Project) build() {
 	p.stage = stageBuild
+	vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
 
-	buildCmd := &exec.Cmd{
+	output, err := runCmd(p.Build, p.workingDir())
-		Path: p.Build,
-		Dir:  p.verDir(),
-	}
-
-	output, err := buildCmd.Output()
 
 	if p.errHandled(err) {
 		return
@@ -225,13 +252,9 @@ func (p *Project) build() {
 // test runs the test scripts
 func (p *Project) test() {
 	p.stage = stageTest
+	vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
 
-	testCmd := &exec.Cmd{
+	output, err := runCmd(p.Test, p.workingDir())
-		Path: p.Test,
-		Dir:  p.verDir(),
-	}
-
-	output, err := testCmd.Output()
 
 	if p.errHandled(err) {
 		return
@@ -249,13 +272,9 @@ func (p *Project) test() {
 // release runs the release scripts and builds the release file
 func (p *Project) release() {
 	p.stage = stageRelease
+	vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
 
-	releaseCmd := &exec.Cmd{
+	output, err := runCmd(p.Release, p.workingDir())
-		Path: p.Release,
-		Dir:  p.verDir(),
-	}
-
-	output, err := releaseCmd.Output()
 
 	if p.errHandled(err) {
 		return
@@ -266,7 +285,7 @@ func (p *Project) release() {
 	}
 
 	//get release file
-	f, err := os.Open(filepath.Join(p.verDir(), p.ReleaseFile))
+	f, err := os.Open(filepath.Join(p.workingDir(), p.ReleaseFile))
 	if p.errHandled(err) {
 		return
 	}
@@ -280,4 +299,12 @@ func (p *Project) release() {
 		return
 	}
 
+	//build successfull, remove working dir
+	p.errHandled(os.RemoveAll(p.workingDir()))
+
+	if p.errHandled(p.ds.Close()) {
+		return
+	}
+
+	vlog("Project: %s Version %s built, tested, and released successfully.\n", p.id(), p.version)
 }

datastore/ds.go

@@ -64,7 +64,7 @@ func Open(filename string) (*Store, error) {
 
 // Close closes the bolt datastore
 func (ds *Store) Close() error {
-	return ds.Close()
+	return ds.bolt.Close()
 }
 
 func (ds *Store) get(bucket string, key TimeKey, result interface{}) error {

datastore/log.go

@@ -34,7 +34,7 @@ func (ds *Store) AddLog(version, stage, entry string) error {
 	return ds.put(bucketLog, key, data)
 }
 
-// LatestVersion returns the latest version for the current project
+// LatestVersion returns the latest version (successful or otherwise) for the current project
 func (ds *Store) LatestVersion() (string, error) {
 	version := ""
 

exec.go

@@ -0,0 +1,33 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+func runCmd(cmd, dir string) ([]byte, error) {
+	s := strings.Fields(cmd)
+
+	var args []string
+
+	if len(s) > 1 {
+		args = s[1:]
+	}
+
+	ec := exec.Command(s[0], args...)
+
+	ec.Dir = dir
+
+	vlog("Executing command: %s in dir %s\n", cmd, dir)
+
+	result, err := ec.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("%s:\n%s", err, result)
+	}
+	return result, nil
+}

log.go

@@ -0,0 +1,13 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import "log"
+
+func vlog(format string, v ...interface{}) {
+	if verbose {
+		log.Printf(format, v...)
+	}
+}

main.go

@@ -5,8 +5,10 @@
 package main
 
 import (
+	"flag"
 	"log"
 	"os"
+	"os/signal"
 	"path/filepath"
 
 	"git.townsourced.com/config"
@@ -16,23 +18,46 @@ import (
 var (
 	projectDir = "./projects" // /etc/ironsmith/
 	dataDir    = "./data"     // /var/ironsmith/
-	address    = "http://localhost:8026"
+	address    = ":8026"
 	certFile   = ""
 	keyFile    = ""
 )
 
+//flags
+var (
+	verbose = false
+)
+
+func init() {
+	flag.BoolVar(&verbose, "v", false, "Verbose prints to stdOut every command and stage as it processes")
+
+	//Capture program shutdown, to make sure everything shuts down nicely
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		for sig := range c {
+			if sig == os.Interrupt {
+				projects.closeAll()
+				os.Exit(0)
+			}
+		}
+	}()
+}
+
 func main() {
+	flag.Parse()
+
 	settingPaths := config.StandardFileLocations("ironsmith/settings.json")
-	log.Println("IronSmith will use settings files in the following locations (in order of priority):")
+	vlog("IronSmith will use settings files in the following locations (in order of priority):\n")
 	for i := range settingPaths {
-		log.Println("\t", settingPaths[i])
+		vlog("\t%s\n", settingPaths[i])
 	}
 	cfg, err := config.LoadOrCreate(settingPaths...)
 	if err != nil {
 		log.Fatalf("Error loading or creating IronSmith settings file: %s", err)
 	}
 
-	log.Printf("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
+	vlog("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
 
 	projectDir = cfg.String("projectDir", projectDir)
 	dataDir = cfg.String("dataDir", dataDir)
@@ -40,6 +65,9 @@ func main() {
 	certFile = cfg.String("certFile", certFile)
 	keyFile = cfg.String("keyFile", keyFile)
 
+	vlog("Project Definition Directory: %s\n", projectDir)
+	vlog("Project Data Directory: %s\n", dataDir)
+
 	//prep dirs
 	err = os.MkdirAll(filepath.Join(projectDir, enabledProjectDir), 0777)
 	if err != nil {
@@ -61,5 +89,12 @@ func main() {
 	if err != nil {
 		log.Fatalf("Error loading projects: %s", err)
 	}
-	//start server
+
+	//start web server
+
+	err = startServer()
+	if err != nil {
+		log.Fatalf("Error Starting web server: %s", err)
+	}
+
 }

project.go

@@ -59,6 +59,7 @@ type Project struct {
 	ds       *datastore.Store
 	stage    string
 	version  string
+	hash     string
 }
 
 const projectTemplateFilename = "template.project.json"
@@ -66,12 +67,12 @@ const projectTemplateFilename = "template.project.json"
 var projectTemplate = &Project{
 	Name:    "Template Project",
 	Fetch:   "git clone root@git.townsourced.com:tshannon/ironsmith.git .",
-	Build:   "sh ./ironsmith/build.sh",
+	Build:   "go build -o ironsmith",
-	Test:    "sh ./ironsmith/test.sh",
+	Test:    "go test ./...",
-	Release: "sh ./ironsmith/release.sh",
+	Release: "tar -czf release.tar.gz ironsmith",
 	Version: "git describe --tags --long",
 
-	ReleaseFile:  `json:"./ironsmith/release.tar.gz"`,
+	ReleaseFile:  "release.tar.gz",
 	PollInterval: "15m",
 }
 
@@ -79,6 +80,7 @@ func prepTemplateProject() error {
 	filename := filepath.Join(projectDir, projectTemplateFilename)
 	_, err := os.Stat(filename)
 	if os.IsNotExist(err) {
+		vlog("Creating template project file in %s", filename)
 		f, err := os.Create(filename)
 		defer func() {
 			if cerr := f.Close(); cerr != nil && err == nil {
@@ -117,6 +119,7 @@ var projects = projectList{
 }
 
 func (p *projectList) load() error {
+	vlog("Loading projects from the enabled definitions in %s\n", filepath.Join(projectDir, enabledProjectDir))
 	dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
 	defer func() {
 		if cerr := dir.Close(); cerr != nil && err == nil {
@@ -153,6 +156,7 @@ func (p *projectList) exists(name string) bool {
 }
 
 func (p *projectList) add(name string) {
+	vlog("Adding project %s to the project list.\n", name)
 	p.Lock()
 	defer p.Unlock()
 
@@ -181,11 +185,22 @@ func (p *projectList) removeMissing(names []string) {
 			}
 		}
 		if !found {
+			vlog("Removing project %s from the project list, because the project file was removed.\n",
+				i)
 			delete(p.data, i)
 		}
 	}
 }
 
+func (p *projectList) closeAll() {
+	p.RLock()
+	defer p.RUnlock()
+
+	for i := range p.data {
+		_ = p.data[i].ds.Close()
+	}
+}
+
 // startProjectLoader polls for new projects
 func startProjectLoader() {
 	dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
@@ -196,13 +211,13 @@ func startProjectLoader() {
 	}()
 
 	if err != nil {
-		log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", dir, err)
+		log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", projectDir, err)
 		return
 	}
 
 	files, err := dir.Readdir(0)
 	if err != nil {
-		log.Printf("Error in startProjectLoader reading the dir %s: %s\n", dir, err)
+		log.Printf("Error in startProjectLoader reading the dir %s: %s\n", projectDir, err)
 		return
 	}
 

server.go

@@ -0,0 +1,77 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package main
+
+import "net/http"
+
+var webRoot *http.ServeMux
+
+func startServer() error {
+	var err error
+
+	routes()
+
+	server := &http.Server{
+		Handler: webRoot,
+		Addr:    address,
+	}
+
+	if certFile == "" || keyFile == "" {
+		err = server.ListenAndServe()
+	} else {
+		server.Addr = address
+		err = server.ListenAndServeTLS(certFile, keyFile)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type methodHandler struct {
+	get    http.HandlerFunc
+	post   http.HandlerFunc
+	put    http.HandlerFunc
+	delete http.HandlerFunc
+}
+
+func (m *methodHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if m.get == nil {
+		m.get = http.NotFound
+	}
+	if m.post == nil {
+		m.post = http.NotFound
+	}
+	if m.put == nil {
+		m.put = http.NotFound
+	}
+	if m.delete == nil {
+		m.delete = http.NotFound
+	}
+	switch r.Method {
+	case "GET":
+		m.get(w, r)
+		return
+	case "POST":
+		m.post(w, r)
+		return
+	case "PUT":
+		m.put(w, r)
+		return
+	case "DELETE":
+		m.delete(w, r)
+		return
+	default:
+		http.NotFound(w, r)
+		return
+	}
+}
+
+func routes() {
+	webRoot = http.NewServeMux()
+
+}