gobs webhook test

instead of anytime that it's not polled

README.md

@@ -14,11 +14,24 @@ You'll setup a project which will need the following information:
 2. Script to build the repository
 3. Script to test the repository
 4. Script to build the release file
-5. Path to the release file
+5. Path to the release file / can also be a script that returns a file name
 6. Script to set release name / version
 
+An optional set of environment strings can be set to define the environment in which the scripts run.
+```
+"environment": [
+"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/local/go/bin",
+"GOPATH=@dir"
+]
+```
+
 Projects will be defined in a project.json file for now.  I may add a web interface later.
 
+@dir in any of the script strings or environment entries will be replaced with an absolute path to the current working directory of the specific version being worked on.
+```
+sh ./build.sh @dir
+```
+
 
 Ironsmith will take the information for the defined project above and do the following
 
@@ -26,7 +39,8 @@ Ironsmith will take the information for the defined project above and do the fol
 2. Change to that directory
 2. Create a bolt DB file for the project to keep a log of all the builds
 3. Run an initial pull of the repository using the pull script
-4. If pull succeeds, Run the Build Scripts
+4. Run version script
+4. If pull is a new version, then Run the Build Scripts
 5. If build succeeds, run the test scripts
 6. If test succeeds, run the release scripts
 7. Load the release file into project release folder with the release name

build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+go-bindata web/... && go build -a -o ironsmith

cycle.go

@@ -15,7 +15,7 @@ import (
 	"strings"
 	"time"
 
-	"git.townsourced.com/ironsmith/datastore"
+	"git.townsourced.com/townsourced/ironsmith/datastore"
 )
 
 /*
@@ -27,12 +27,13 @@ Project life cycle:
 
 // load is the beginning of the cycle.  Loads / reloads the project file to make sure that the scripts are up-to-date
 // call's fetch and triggers the next poll if one exists
-func (p *Project) load() {
+func (p *Project) load(forceBuild bool) {
 	p.processing.Lock() // ensure only one cycle is running at a time per project
 	defer p.processing.Unlock()
 
 	p.setStage(stageLoad)
-	p.setVersion("")
+	p.setVersion("Version not yet set")
+	p.start = time.Time{}
 
 	if p.filename == "" {
 		p.errHandled(errors.New("Invalid project file name"))
@@ -63,25 +64,27 @@ func (p *Project) load() {
 
 	p.setData(new)
 
-	p.fetch()
+	p.fetch(forceBuild)
 
 	p.setStage(stageWait)
 
 	//full cycle completed
+	p.errHandled(p.ds.TrimVersions(p.MaxVersions))
 
 	if p.poll > 0 {
 		//start polling
-		go func() {
-			time.AfterFunc(p.poll, p.load)
-		}()
+		time.AfterFunc(p.poll, func() {
+			p.load(false)
+		})
 	}
 }
 
 // fetch first runs the fetch script into a temporary directory
 // then it runs the version script in the temp directory to see if there is a newer version of the
 // fetched code, if there is then the temp dir is renamed to the version name
-func (p *Project) fetch() {
+func (p *Project) fetch(forceBuild bool) {
 	p.setStage(stageFetch)
+	p.start = time.Now()
 
 	if p.Fetch == "" {
 		return
@@ -94,13 +97,13 @@ func (p *Project) fetch() {
 	}
 
 	//fetch project
-	fetchResult, err := runCmd(p.Fetch, tempDir)
+	fetchResult, err := runCmd(p.Fetch, tempDir, p.Environment)
 	if p.errHandled(err) {
 		return
 	}
 
 	// fetched succesfully, determine version
-	version, err := runCmd(p.Version, tempDir)
+	version, err := runCmd(p.Version, tempDir, p.Environment)
 
 	if p.errHandled(err) {
 		return
@@ -108,18 +111,20 @@ func (p *Project) fetch() {
 
 	p.setVersion(strings.TrimSpace(string(version)))
 
-	// check if this specific version has attempted a build yet
-	lVer, err := p.ds.LastVersion(stageBuild)
-	if err != datastore.ErrNotFound && p.errHandled(err) {
-		return
-	}
+	if !forceBuild {
+		// if not forced build, then check if this specific version has attempted a build yet
+		lVer, err := p.ds.LastVersion(stageBuild)
+		if err != datastore.ErrNotFound && p.errHandled(err) {
+			return
+		}
 
-	if p.version == "" || p.version == lVer.Version {
-		// no new build clean up temp dir
-		p.errHandled(os.RemoveAll(tempDir))
+		if p.version == "" || p.version == lVer.Version {
+			// no new build clean up temp dir
+			p.errHandled(os.RemoveAll(tempDir))
 
-		vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
-		return
+			vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
+			return
+		}
 	}
 
 	//remove any existing data that matches version hash
@@ -152,7 +157,7 @@ func (p *Project) build() {
 		return
 	}
 
-	output, err := runCmd(p.Build, p.workingDir())
+	output, err := runCmd(p.Build, p.workingDir(), p.Environment)
 
 	if p.errHandled(err) {
 		return
@@ -173,7 +178,7 @@ func (p *Project) test() {
 	if p.Test == "" {
 		return
 	}
-	output, err := runCmd(p.Test, p.workingDir())
+	output, err := runCmd(p.Test, p.workingDir(), p.Environment)
 
 	if p.errHandled(err) {
 		return
@@ -195,7 +200,7 @@ func (p *Project) release() {
 		return
 	}
 
-	output, err := runCmd(p.Release, p.workingDir())
+	output, err := runCmd(p.Release, p.workingDir(), p.Environment)
 
 	if p.errHandled(err) {
 		return
@@ -216,14 +221,15 @@ func (p *Project) release() {
 		return
 	}
 
-	if p.errHandled(p.ds.AddRelease(p.version, p.ReleaseFile, buff)) {
+	if p.errHandled(p.ds.AddRelease(p.version, filepath.Base(p.ReleaseFile), buff)) {
 		return
 	}
 
 	p.setStage(stageReleased)
 
 	if p.errHandled(p.ds.AddLog(p.version, p.stage,
-		fmt.Sprintf("Project %s Version %s built, tested, and released successfully.\n", p.id(), p.version))) {
+		fmt.Sprintf("Project %s Version %s built, tested, and released successfully and took %s.\n", p.id(), p.version,
+			time.Now().Sub(p.start)))) {
 		return
 	}
 

datastore/ds.go

@@ -67,6 +67,83 @@ func (ds *Store) Close() error {
 	return ds.bolt.Close()
 }
 
+// TrimVersions Removes versions from the datastore file until it reaches the maxVersions count
+func (ds *Store) TrimVersions(maxVersions int) error {
+	if maxVersions <= 0 {
+		// no max set
+		return nil
+	}
+
+	versions, err := ds.Versions()
+	if err != nil {
+		return err
+	}
+
+	if len(versions) <= maxVersions {
+		return nil
+	}
+
+	remove := versions[maxVersions:]
+
+	for i := range remove {
+		err = ds.deleteVersion(remove[i].Version)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// removes the earliest instance of a specific version
+func (ds *Store) deleteVersion(version string) error {
+	return ds.bolt.Update(func(tx *bolt.Tx) error {
+		// remove all logs for this version
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			lg := &Log{}
+
+			err := json.Unmarshal(v, lg)
+			if err != nil {
+				return err
+			}
+
+			if lg.Version != version {
+				break
+			}
+
+			err = c.Delete()
+			if err != nil {
+				return err
+			}
+		}
+
+		// remove all releases for this version
+		release, err := ds.Release(version)
+		if err == ErrNotFound {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		err = tx.Bucket([]byte(bucketReleases)).Delete(release.FileKey.Bytes())
+		if err != nil {
+			return err
+		}
+
+		// remove release file for this version
+		err = tx.Bucket([]byte(bucketFiles)).Delete(release.FileKey.Bytes())
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+}
+
 func (ds *Store) get(bucket string, key []byte, result interface{}) error {
 	return ds.bolt.View(func(tx *bolt.Tx) error {
 		dsValue := tx.Bucket([]byte(bucket)).Get(key)
@@ -93,9 +170,3 @@ func (ds *Store) put(bucket string, key []byte, value interface{}) error {
 		return tx.Bucket([]byte(bucket)).Put(key, dsValue)
 	})
 }
-
-func (ds *Store) delete(bucket string, key []byte) error {
-	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		return tx.Bucket([]byte(bucket)).Delete(key)
-	})
-}

datastore/releases.go

@@ -28,13 +28,13 @@ const (
 
 // AddRelease adds a new Release
 func (ds *Store) AddRelease(version, fileName string, fileData []byte) error {
-	fileKey := NewTimeKey()
+	key := NewTimeKey()
 
 	r := &Release{
-		When:     fileKey.Time(),
+		When:     key.Time(),
 		Version:  version,
 		FileName: fileName,
-		FileKey:  fileKey,
+		FileKey:  key,
 	}
 
 	dsValue, err := json.Marshal(r)
@@ -43,12 +43,12 @@ func (ds *Store) AddRelease(version, fileName string, fileData []byte) error {
 	}
 
 	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		err = tx.Bucket([]byte(bucketReleases)).Put([]byte(version), dsValue)
+		err = tx.Bucket([]byte(bucketReleases)).Put(key.Bytes(), dsValue)
 		if err != nil {
 			return err
 		}
 
-		return tx.Bucket([]byte(bucketFiles)).Put(fileKey.Bytes(), fileData)
+		return tx.Bucket([]byte(bucketFiles)).Put(key.Bytes(), fileData)
 	})
 }
 
@@ -80,10 +80,27 @@ func (ds *Store) ReleaseFile(fileKey TimeKey) ([]byte, error) {
 // Release gets the release record for a specific version
 func (ds *Store) Release(version string) (*Release, error) {
 	r := &Release{}
-	err := ds.get(bucketReleases, []byte(version), r)
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketReleases)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			err := json.Unmarshal(v, r)
+			if err != nil {
+				return err
+			}
+
+			if r.Version == version {
+				return nil
+			}
+		}
+
+		return ErrNotFound
+	})
+
 	if err != nil {
 		return nil, err
 	}
+
 	return r, nil
 }
 
@@ -119,9 +136,8 @@ func (ds *Store) LastRelease() (*Release, error) {
 	r := &Release{}
 
 	err := ds.bolt.View(func(tx *bolt.Tx) error {
-		c := tx.Bucket([]byte(bucketReleases)).Cursor()
+		_, v := tx.Bucket([]byte(bucketReleases)).Cursor().Last()
 
-		_, v := c.Last()
 		if v == nil {
 			return ErrNotFound
 		}

error.go

@@ -11,7 +11,7 @@ import (
 	"log"
 	"net/http"
 
-	"git.townsourced.com/ironsmith/datastore"
+	"git.townsourced.com/townsourced/ironsmith/datastore"
 )
 
 const (

exec.go

@@ -6,12 +6,18 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 )
 
-func runCmd(cmd, dir string) ([]byte, error) {
-	s := strings.Fields(cmd)
+func runCmd(cmd, dir string, env []string) ([]byte, error) {
+	s := strings.Fields(strings.Replace(cmd, "@dir", dir, -1))
+
+	for i := range env {
+		env[i] = strings.Replace(env[i], "@dir", dir, -1)
+	}
 
 	var args []string
 
@@ -19,15 +25,74 @@ func runCmd(cmd, dir string) ([]byte, error) {
 		args = s[1:]
 	}
 
-	ec := exec.Command(s[0], args...)
+	name := s[0]
+	ec := &exec.Cmd{
+		Path: name,
+		Args: append([]string{name}, args...),
+		Dir:  dir,
+		Env:  env,
+	}
 
-	ec.Dir = dir
+	if filepath.Base(name) == name {
+		lp, err := lookPath(name, env)
+		if err != nil {
+			return nil, err
+		}
+		ec.Path = lp
+	}
 
 	vlog("Executing command: %s in dir %s\n", cmd, dir)
 
 	result, err := ec.CombinedOutput()
 	if err != nil {
-		return nil, fmt.Errorf("%s", result)
+		return nil, fmt.Errorf("%s\n%s", err, result)
 	}
 	return result, nil
 }
+
+// similar to os/exec.LookPath, except it checks if the passed in
+// custom environment includes a path definitions and uses that path instead
+// note this probably only works on unix, that's all I care about for now
+func lookPath(file string, env []string) (string, error) {
+	if strings.Contains(file, "/") {
+		err := findExecutable(file)
+		if err == nil {
+			return file, nil
+		}
+		return "", &exec.Error{Name: file, Err: err}
+	}
+
+	for i := range env {
+		if strings.HasPrefix(env[i], "PATH=") {
+			pathenv := env[i][5:]
+			if pathenv == "" {
+				return "", &exec.Error{Name: file, Err: exec.ErrNotFound}
+			}
+			for _, dir := range strings.Split(pathenv, ":") {
+				if dir == "" {
+					// Unix shell semantics: path element "" means "."
+					dir = "."
+				}
+				path := dir + "/" + file
+				if err := findExecutable(path); err == nil {
+					return path, nil
+				}
+			}
+			return "", &exec.Error{Name: file, Err: exec.ErrNotFound}
+		}
+	}
+
+	return exec.LookPath(file)
+}
+
+func findExecutable(file string) error {
+	d, err := os.Stat(file)
+	if err != nil {
+		return err
+	}
+	if m := d.Mode(); !m.IsDir() && m&0111 != 0 {
+		return nil
+	}
+
+	return os.ErrPermission
+}

main.go

@@ -11,7 +11,7 @@ import (
 	"os/signal"
 	"path/filepath"
 
-	"git.townsourced.com/config"
+	"git.townsourced.com/townsourced/config"
 )
 
 //settings

project.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"git.townsourced.com/ironsmith/datastore"
+	"git.townsourced.com/townsourced/ironsmith/datastore"
 )
 
 const (
@@ -48,6 +48,8 @@ The project lifecycle goes like this, each step calling the next if successful
 type Project struct {
 	Name string `json:"name"` // name of the project
 
+	Environment []string `json:"environment"` // Environment for each of the scripts below, if empty will use the current processes environment
+
 	Fetch   string `json:"fetch"`   //Script to fetch the latest project code into the current directory
 	Build   string `json:"build"`   //Script to build the latest project code
 	Test    string `json:"test"`    //Script to test the latest project code
@@ -58,6 +60,7 @@ type Project struct {
 	ReleaseFile   string `json:"releaseFile"`
 	PollInterval  string `json:"pollInterval,omitempty"`  // if not poll interval is specified, this project is trigger only
 	TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call
+	MaxVersions   int    `json:"maxVersions,omitempty"`   // Max number of versions to keep in the project datastore
 
 	filename string
 	poll     time.Duration
@@ -66,6 +69,7 @@ type Project struct {
 	status   string
 	version  string
 	hash     string
+	start    time.Time // the last start time of the latest cycle
 
 	sync.RWMutex
 	processing sync.Mutex
@@ -273,6 +277,7 @@ func (p *Project) setData(new *Project) {
 	defer p.Unlock()
 
 	p.Name = new.Name
+	p.Environment = new.Environment
 
 	p.Fetch = new.Fetch
 	p.Build = new.Build
@@ -283,6 +288,7 @@ func (p *Project) setData(new *Project) {
 	p.ReleaseFile = new.ReleaseFile
 	p.PollInterval = new.PollInterval
 	p.TriggerSecret = new.TriggerSecret
+	p.MaxVersions = new.MaxVersions
 
 	if p.PollInterval != "" {
 		var err error
@@ -320,6 +326,8 @@ var projectTemplate = &Project{
 
 	ReleaseFile:  "release.tar.gz",
 	PollInterval: "15m",
+
+	MaxVersions: 100,
 }
 
 func prepTemplateProject() error {
@@ -419,7 +427,7 @@ func (p *projectList) add(name string) {
 			log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err)
 			return
 		}
-		prj.load()
+		prj.load(false)
 	}()
 }
 

web/index.html

@@ -163,10 +163,10 @@
 		<tr>
 			<th>Project</th>
 			<th>Status</th>
-			<th>Last Release</th>
-			<th>Last Release File</th>
 			<th>Last Version</th>
 			<th>Last Log</th>
+			<th>Last Release</th>
+			<th>Last Release File</th>
 		</tr>
 	</thead>
 	<tbody>
@@ -174,6 +174,10 @@
 			<tr title="{{formatDate(.lastLog.when)}}">
 				<td><a href="/project/{{.id}}/">{{.name}}</a></td>
 				<td>{{.status}}</td>
+				<td>
+					<a href="/project/{{.id}}/{{.lastLog.version}}">{{.lastLog.version}}</a>
+				</td>
+				<td title="{{.lastLog.log}}">{{#if .lastLog.log && .lastLog.log.length > 150}}{{.lastLog.log.substring(0,150)}}...{{else}}{{.lastLog.log}}{{/if}}</td>
 				<td>
 					<a href="/project/{{.id}}/{{.releaseVersion}}">{{.releaseVersion}}</a>
 				</td>
@@ -184,10 +188,6 @@
 						No release file available
 					{{/if}}
 				</td>
-				<td>
-					<a href="/project/{{.id}}/{{.lastLog.version}}">{{.lastLog.version}}</a>
-				</td>
-				<td title="{{.lastLog.log}}">{{#if .lastLog.log}}{{.lastLog.log.substring(0,100)}}{{/if}}</td>
 			</tr>
 		{{/projects}}
 	</tbody>
@@ -213,7 +213,7 @@
 					<a href="/project/{{project.id}}/{{.version}}">{{.version}}</a>
 				</td>
 				<td>{{.stage}}</td>
-				<td title="{{.log}}">{{#if .log}}{{.log.substring(0,100)}}{{/if}}</td>
+				<td title="{{.log}}">{{#if .log && .log.length > 150}}{{.log.substring(0,150)}}...{{else}}{{.log}}{{/if}}</td>
 				<td>
 					{{#if releases[project.id + .version]}}
 						<a href="/release/{{project.id}}/{{.version}}?file">{{releases[project.id + .version].fileName}}</a>	

web/js/index.js

@@ -135,7 +135,11 @@ Ractive.DEBUG = false;
     function getVersion(id, version) {
         get("/log/" + id + "/" + version,
             function(result) {
-                r.set("version", version);
+                if (!result.data || !result.data.length || !result.data[0].version) {
+                    r.set("version", version);
+                } else {
+                    r.set("version", result.data[0].version);
+                }
                 r.set("stages", result.data);
             },
             function(result) {
@@ -171,6 +175,8 @@ Ractive.DEBUG = false;
         //statuses 
         if (project.stage != "waiting") {
             project.status = project.stage;
+        } else if (!project.lastLog || !project.lastLog.version) {
+            project.status = "waiting";
         } else if (project.lastLog.version.trim() == project.releaseVersion.trim()) {
             project.status = "Successfully Released";
         } else {

webProject.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 	"time"
 
-	"git.townsourced.com/ironsmith/datastore"
+	"git.townsourced.com/townsourced/ironsmith/datastore"
 )
 
 // /path/<project-id>/<version>/<stage>
@@ -259,6 +259,6 @@ func triggerPost(w http.ResponseWriter, r *http.Request) {
 	}
 
 	go func() {
-		project.load()
+		project.load(true)
 	}()
 }