gobs webhook test

instead of anytime that it's not polled

README.md

@@ -14,7 +14,7 @@ You'll setup a project which will need the following information:
 2. Script to build the repository
 3. Script to test the repository
 4. Script to build the release file
-5. Path to the release file
+5. Path to the release file / can also be a script that returns a file name
 6. Script to set release name / version
 
 An optional set of environment strings can be set to define the environment in which the scripts run.

build.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-go-bindata web/... && go build -a -v -o ironsmith
+go-bindata web/... && go build -a -o ironsmith

cycle.go

@@ -27,12 +27,13 @@ Project life cycle:
 
 // load is the beginning of the cycle.  Loads / reloads the project file to make sure that the scripts are up-to-date
 // call's fetch and triggers the next poll if one exists
-func (p *Project) load() {
+func (p *Project) load(forceBuild bool) {
 	p.processing.Lock() // ensure only one cycle is running at a time per project
 	defer p.processing.Unlock()
 
 	p.setStage(stageLoad)
-	p.setVersion("")
+	p.setVersion("Version not yet set")
+	p.start = time.Time{}
 
 	if p.filename == "" {
 		p.errHandled(errors.New("Invalid project file name"))
@@ -63,25 +64,27 @@ func (p *Project) load() {
 
 	p.setData(new)
 
-	p.fetch()
+	p.fetch(forceBuild)
 
 	p.setStage(stageWait)
 
 	//full cycle completed
+	p.errHandled(p.ds.TrimVersions(p.MaxVersions))
 
 	if p.poll > 0 {
 		//start polling
-		go func() {
-			time.AfterFunc(p.poll, p.load)
-		}()
+		time.AfterFunc(p.poll, func() {
+			p.load(false)
+		})
 	}
 }
 
 // fetch first runs the fetch script into a temporary directory
 // then it runs the version script in the temp directory to see if there is a newer version of the
 // fetched code, if there is then the temp dir is renamed to the version name
-func (p *Project) fetch() {
+func (p *Project) fetch(forceBuild bool) {
 	p.setStage(stageFetch)
+	p.start = time.Now()
 
 	if p.Fetch == "" {
 		return
@@ -108,18 +111,20 @@ func (p *Project) fetch() {
 
 	p.setVersion(strings.TrimSpace(string(version)))
 
-	// check if this specific version has attempted a build yet
-	lVer, err := p.ds.LastVersion(stageBuild)
-	if err != datastore.ErrNotFound && p.errHandled(err) {
-		return
-	}
+	if !forceBuild {
+		// if not forced build, then check if this specific version has attempted a build yet
+		lVer, err := p.ds.LastVersion(stageBuild)
+		if err != datastore.ErrNotFound && p.errHandled(err) {
+			return
+		}
 
-	if p.version == "" || p.version == lVer.Version {
-		// no new build clean up temp dir
-		p.errHandled(os.RemoveAll(tempDir))
+		if p.version == "" || p.version == lVer.Version {
+			// no new build clean up temp dir
+			p.errHandled(os.RemoveAll(tempDir))
 
-		vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
-		return
+			vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
+			return
+		}
 	}
 
 	//remove any existing data that matches version hash
@@ -216,14 +221,15 @@ func (p *Project) release() {
 		return
 	}
 
-	if p.errHandled(p.ds.AddRelease(p.version, p.ReleaseFile, buff)) {
+	if p.errHandled(p.ds.AddRelease(p.version, filepath.Base(p.ReleaseFile), buff)) {
 		return
 	}
 
 	p.setStage(stageReleased)
 
 	if p.errHandled(p.ds.AddLog(p.version, p.stage,
-		fmt.Sprintf("Project %s Version %s built, tested, and released successfully.\n", p.id(), p.version))) {
+		fmt.Sprintf("Project %s Version %s built, tested, and released successfully and took %s.\n", p.id(), p.version,
+			time.Now().Sub(p.start)))) {
 		return
 	}
 

datastore/ds.go

@@ -67,6 +67,83 @@ func (ds *Store) Close() error {
 	return ds.bolt.Close()
 }
 
+// TrimVersions Removes versions from the datastore file until it reaches the maxVersions count
+func (ds *Store) TrimVersions(maxVersions int) error {
+	if maxVersions <= 0 {
+		// no max set
+		return nil
+	}
+
+	versions, err := ds.Versions()
+	if err != nil {
+		return err
+	}
+
+	if len(versions) <= maxVersions {
+		return nil
+	}
+
+	remove := versions[maxVersions:]
+
+	for i := range remove {
+		err = ds.deleteVersion(remove[i].Version)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// removes the earliest instance of a specific version
+func (ds *Store) deleteVersion(version string) error {
+	return ds.bolt.Update(func(tx *bolt.Tx) error {
+		// remove all logs for this version
+		c := tx.Bucket([]byte(bucketLog)).Cursor()
+
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			lg := &Log{}
+
+			err := json.Unmarshal(v, lg)
+			if err != nil {
+				return err
+			}
+
+			if lg.Version != version {
+				break
+			}
+
+			err = c.Delete()
+			if err != nil {
+				return err
+			}
+		}
+
+		// remove all releases for this version
+		release, err := ds.Release(version)
+		if err == ErrNotFound {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		err = tx.Bucket([]byte(bucketReleases)).Delete(release.FileKey.Bytes())
+		if err != nil {
+			return err
+		}
+
+		// remove release file for this version
+		err = tx.Bucket([]byte(bucketFiles)).Delete(release.FileKey.Bytes())
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+}
+
 func (ds *Store) get(bucket string, key []byte, result interface{}) error {
 	return ds.bolt.View(func(tx *bolt.Tx) error {
 		dsValue := tx.Bucket([]byte(bucket)).Get(key)
@@ -93,9 +170,3 @@ func (ds *Store) put(bucket string, key []byte, value interface{}) error {
 		return tx.Bucket([]byte(bucket)).Put(key, dsValue)
 	})
 }
-
-func (ds *Store) delete(bucket string, key []byte) error {
-	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		return tx.Bucket([]byte(bucket)).Delete(key)
-	})
-}

datastore/releases.go

@@ -28,13 +28,13 @@ const (
 
 // AddRelease adds a new Release
 func (ds *Store) AddRelease(version, fileName string, fileData []byte) error {
-	fileKey := NewTimeKey()
+	key := NewTimeKey()
 
 	r := &Release{
-		When:     fileKey.Time(),
+		When:     key.Time(),
 		Version:  version,
 		FileName: fileName,
-		FileKey:  fileKey,
+		FileKey:  key,
 	}
 
 	dsValue, err := json.Marshal(r)
@@ -43,12 +43,12 @@ func (ds *Store) AddRelease(version, fileName string, fileData []byte) error {
 	}
 
 	return ds.bolt.Update(func(tx *bolt.Tx) error {
-		err = tx.Bucket([]byte(bucketReleases)).Put([]byte(version), dsValue)
+		err = tx.Bucket([]byte(bucketReleases)).Put(key.Bytes(), dsValue)
 		if err != nil {
 			return err
 		}
 
-		return tx.Bucket([]byte(bucketFiles)).Put(fileKey.Bytes(), fileData)
+		return tx.Bucket([]byte(bucketFiles)).Put(key.Bytes(), fileData)
 	})
 }
 
@@ -80,10 +80,27 @@ func (ds *Store) ReleaseFile(fileKey TimeKey) ([]byte, error) {
 // Release gets the release record for a specific version
 func (ds *Store) Release(version string) (*Release, error) {
 	r := &Release{}
-	err := ds.get(bucketReleases, []byte(version), r)
+	err := ds.bolt.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte(bucketReleases)).Cursor()
+
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			err := json.Unmarshal(v, r)
+			if err != nil {
+				return err
+			}
+
+			if r.Version == version {
+				return nil
+			}
+		}
+
+		return ErrNotFound
+	})
+
 	if err != nil {
 		return nil, err
 	}
+
 	return r, nil
 }
 
@@ -119,9 +136,8 @@ func (ds *Store) LastRelease() (*Release, error) {
 	r := &Release{}
 
 	err := ds.bolt.View(func(tx *bolt.Tx) error {
-		c := tx.Bucket([]byte(bucketReleases)).Cursor()
+		_, v := tx.Bucket([]byte(bucketReleases)).Cursor().Last()
 
-		_, v := c.Last()
 		if v == nil {
 			return ErrNotFound
 		}

exec.go

@@ -59,14 +59,14 @@ func lookPath(file string, env []string) (string, error) {
 		if err == nil {
 			return file, nil
 		}
-		return "", &exec.Error{file, err}
+		return "", &exec.Error{Name: file, Err: err}
 	}
 
 	for i := range env {
 		if strings.HasPrefix(env[i], "PATH=") {
 			pathenv := env[i][5:]
 			if pathenv == "" {
-				return "", &exec.Error{file, exec.ErrNotFound}
+				return "", &exec.Error{Name: file, Err: exec.ErrNotFound}
 			}
 			for _, dir := range strings.Split(pathenv, ":") {
 				if dir == "" {
@@ -78,7 +78,7 @@ func lookPath(file string, env []string) (string, error) {
 					return path, nil
 				}
 			}
-			return "", &exec.Error{file, exec.ErrNotFound}
+			return "", &exec.Error{Name: file, Err: exec.ErrNotFound}
 		}
 	}
 

project.go

@@ -60,6 +60,7 @@ type Project struct {
 	ReleaseFile   string `json:"releaseFile"`
 	PollInterval  string `json:"pollInterval,omitempty"`  // if not poll interval is specified, this project is trigger only
 	TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call
+	MaxVersions   int    `json:"maxVersions,omitempty"`   // Max number of versions to keep in the project datastore
 
 	filename string
 	poll     time.Duration
@@ -68,6 +69,7 @@ type Project struct {
 	status   string
 	version  string
 	hash     string
+	start    time.Time // the last start time of the latest cycle
 
 	sync.RWMutex
 	processing sync.Mutex
@@ -286,6 +288,7 @@ func (p *Project) setData(new *Project) {
 	p.ReleaseFile = new.ReleaseFile
 	p.PollInterval = new.PollInterval
 	p.TriggerSecret = new.TriggerSecret
+	p.MaxVersions = new.MaxVersions
 
 	if p.PollInterval != "" {
 		var err error
@@ -323,6 +326,8 @@ var projectTemplate = &Project{
 
 	ReleaseFile:  "release.tar.gz",
 	PollInterval: "15m",
+
+	MaxVersions: 100,
 }
 
 func prepTemplateProject() error {
@@ -422,7 +427,7 @@ func (p *projectList) add(name string) {
 			log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err)
 			return
 		}
-		prj.load()
+		prj.load(false)
 	}()
 }
 

web/index.html

@@ -177,7 +177,7 @@
 				<td>
 					<a href="/project/{{.id}}/{{.lastLog.version}}">{{.lastLog.version}}</a>
 				</td>
-				<td title="{{.lastLog.log}}">{{#if .lastLog.log}}{{.lastLog.log.substring(0,100)}}{{/if}}</td>
+				<td title="{{.lastLog.log}}">{{#if .lastLog.log && .lastLog.log.length > 150}}{{.lastLog.log.substring(0,150)}}...{{else}}{{.lastLog.log}}{{/if}}</td>
 				<td>
 					<a href="/project/{{.id}}/{{.releaseVersion}}">{{.releaseVersion}}</a>
 				</td>
@@ -213,7 +213,7 @@
 					<a href="/project/{{project.id}}/{{.version}}">{{.version}}</a>
 				</td>
 				<td>{{.stage}}</td>
-				<td title="{{.log}}">{{#if .log}}{{.log.substring(0,100)}}{{/if}}</td>
+				<td title="{{.log}}">{{#if .log && .log.length > 150}}{{.log.substring(0,150)}}...{{else}}{{.log}}{{/if}}</td>
 				<td>
 					{{#if releases[project.id + .version]}}
 						<a href="/release/{{project.id}}/{{.version}}?file">{{releases[project.id + .version].fileName}}</a>	

web/js/index.js

@@ -135,7 +135,11 @@ Ractive.DEBUG = false;
     function getVersion(id, version) {
         get("/log/" + id + "/" + version,
             function(result) {
-                r.set("version", version);
+                if (!result.data || !result.data.length || !result.data[0].version) {
+                    r.set("version", version);
+                } else {
+                    r.set("version", result.data[0].version);
+                }
                 r.set("stages", result.data);
             },
             function(result) {
@@ -171,6 +175,8 @@ Ractive.DEBUG = false;
         //statuses 
         if (project.stage != "waiting") {
             project.status = project.stage;
+        } else if (!project.lastLog || !project.lastLog.version) {
+            project.status = "waiting";
         } else if (project.lastLog.version.trim() == project.releaseVersion.trim()) {
             project.status = "Successfully Released";
         } else {

webProject.go

@@ -259,6 +259,6 @@ func triggerPost(w http.ResponseWriter, r *http.Request) {
 	}
 
 	go func() {
-		project.load()
+		project.load(true)
 	}()
 }