townsourced/ironsmith
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed several issues, contineued web work

Browse Source 
Fixed lots of issues with thread saftey and had to rethink some stuff

Fixed order issues with timekeys
Starting to flesh out the web REST API
This commit is contained in:
Tim Shannon 2016-04-06 21:59:24 +00:00
parent 7d2fa0a6ef
commit b9c945ed92
7 changed files with 340 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
cycle.go
View File

@ -5,12 +5,9 @@
package main
import (
"crypto/sha1"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strconv"
@ -20,83 +17,6 @@ import (
"git.townsourced.com/ironsmith/datastore"
)
func (p *Project) errHandled(err error) bool {
if err == nil {
return false
}
vlog("Error in project %s: %s\n", p.id(), err)
if p.ds == nil {
log.Printf("Error in project %s: %s\n", p.id(), err)
return true
}
defer func() {
//clean up version folder if it exists
if p.version != "" {
err = os.RemoveAll(p.workingDir())
if err != nil {
log.Printf("Error deleting the version directory project %s version %s: %s\n",
p.id(), p.version, err)
}
}
}()
lerr := p.ds.AddLog(p.version, p.stage, err.Error())
if lerr != nil {
log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s",
p.id(), err, lerr)
}
return true
}
func (p *Project) id() string {
if p.filename == "" {
panic("invalid project filename")
}
return strings.TrimSuffix(p.filename, filepath.Ext(p.filename))
}
func (p *Project) dir() string {
return filepath.Join(dataDir, p.id())
}
func (p *Project) workingDir() string {
if p.hash == "" {
panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
}
//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
// timestamp on instead would work just fine, but I like having the working dir tied directly to the
// version returned by project script
return filepath.Join(p.dir(), p.hash)
}
// prepData makes sure the project's data folder and data store is created
/*
folder structure
projectDataFolder/<project-name>/<project-version>
*/
func (p *Project) prepData() error {
err := os.MkdirAll(p.dir(), 0777)
if err != nil {
return err
}
p.ds, err = datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith"))
if err != nil {
return err
}
return nil
}
/*
Project life cycle:
(Load Project file) -> (Fetch) -> (Build) -> (Test) -> (Release) - > (Sleep for polling period) ->
@ -107,20 +27,19 @@ Project life cycle:
// load is the beginning of the cycle. Loads / reloads the project file to make sure that the scripts are up-to-date
// call's fetch and triggers the next poll if one exists
func (p *Project) load() {
p.version = ""
p.hash = ""
vlog("Entering %s stage for Project: %s\n", stageLoad, p.id())
p.setStage(stageLoad)
p.setVersion("")
if p.filename == "" {
p.errHandled(errors.New("Invalid project file name"))
return
}
if !projects.exists(p.filename) {
if _, ok := projects.get(p.id()); !ok {
// project has been deleted
// don't continue polling
// move project data to deleted folder with a timestamp
p.close()
p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir,
strconv.FormatInt(time.Now().Unix(), 10), p.id())))
return
@ -131,30 +50,19 @@ func (p *Project) load() {
return
}
if p.errHandled(json.Unmarshal(data, p)) {
new := &Project{}
if p.errHandled(json.Unmarshal(data, new)) {
return
}
p.stage = stageLoad
p.setData(new)
if p.errHandled(p.prepData()) {
if p.errHandled(os.MkdirAll(p.dir(), 0777)) {
return
}
if p.PollInterval != "" {
p.poll, err = time.ParseDuration(p.PollInterval)
if p.errHandled(err) {
p.poll = 0
}
}
p.fetch()
if p.errHandled(p.ds.Close()) {
return
}
p.ds = nil
//full cycle completed
if p.poll > 0 {
@ -167,9 +75,8 @@ func (p *Project) load() {
// then it runs the version script in the temp directory to see if there is a newer version of the
// fetched code, if there is then the temp dir is renamed to the version name
func (p *Project) fetch() {
p.stage = stageFetch
p.setStage(stageFetch)
vlog("Entering %s stage for Project: %s\n", p.stage, p.id())
tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10))
if p.errHandled(os.MkdirAll(tempDir, 0777)) {
@ -189,9 +96,9 @@ func (p *Project) fetch() {
return
}
p.version = strings.TrimSpace(string(version))
p.setVersion(strings.TrimSpace(string(version)))
lVer, err := p.ds.LatestVersion()
lVer, err := p.ds.LastVersion("")
if err != datastore.ErrNotFound && p.errHandled(err) {
return
}
@ -204,8 +111,6 @@ func (p *Project) fetch() {
return
}
p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(p.version)))
//remove any existing data that matches version hash
if p.errHandled(os.RemoveAll(p.workingDir())) {
return
@ -225,14 +130,12 @@ func (p *Project) fetch() {
// continue to build
p.build()
}
// build runs the build scripts to build the project which should result in the a single file
// configured in the ReleaseFile section of the project file
func (p *Project) build() {
p.stage = stageBuild
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
p.setStage(stageBuild)
output, err := runCmd(p.Build, p.workingDir())
@ -250,9 +153,7 @@ func (p *Project) build() {
// test runs the test scripts
func (p *Project) test() {
p.stage = stageTest
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
p.setStage(stageTest)
output, err := runCmd(p.Test, p.workingDir())
if p.errHandled(err) {
@ -265,13 +166,11 @@ func (p *Project) test() {
// Tests passed, onto release
p.release()
}
// release runs the release scripts and builds the release file
func (p *Project) release() {
p.stage = stageRelease
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
p.setStage(stageRelease)
output, err := runCmd(p.Release, p.workingDir())

10
datastore/key.go
View File

@ -29,10 +29,6 @@ func NewTimeKey() TimeKey {
nsec := t.Nanosecond()
return TimeKey{
rBits[0], //random
rBits[1],
rBits[2],
rBits[3],
byte(sec >> 56), // seconds
byte(sec >> 48),
byte(sec >> 40),
@ -45,12 +41,16 @@ func NewTimeKey() TimeKey {
byte(nsec >> 16),
byte(nsec >> 8),
byte(nsec),
rBits[0], //random
rBits[1],
rBits[2],
rBits[3],
}
}
// Time returns the time portion of a timekey
func (k TimeKey) Time() time.Time {
buf := k[4:]
buf := k[:]
sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 |
int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56

63
datastore/log.go
View File

@ -11,11 +11,12 @@ import (
"github.com/boltdb/bolt"
)
type log struct {
When time.Time `json:"when"`
Version string `json:"version"`
Stage string `json:"stage"`
Log string `json:"log"`
// Log is a version log entry for a project
type Log struct {
When time.Time `json:"when,omitempty"`
Version string `json:"version,omitempty"`
Stage string `json:"stage,omitempty"`
Log string `json:"log,omitempty"`
}
const bucketLog = "log"
@ -24,7 +25,7 @@ const bucketLog = "log"
func (ds *Store) AddLog(version, stage, entry string) error {
key := NewTimeKey()
data := &log{
data := &Log{
When: key.Time(),
Version: version,
Stage: stage,
@ -34,23 +35,26 @@ func (ds *Store) AddLog(version, stage, entry string) error {
return ds.put(bucketLog, key, data)
}
// LatestVersion returns the latest version (successful or otherwise) for the current project
func (ds *Store) LatestVersion() (string, error) {
// LastVersion returns the last version in the log for the given stage. If stage is blank,
// then it returns the last of any stage
func (ds *Store) LastVersion(stage string) (string, error) {
version := ""
err := ds.bolt.View(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte(bucketLog)).Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
l := &log{}
for k, v := c.Last(); k != nil; k, v = c.Prev() {
l := &Log{}
err := json.Unmarshal(v, l)
if err != nil {
return err
}
if l.Version != "" {
version = l.Version
return nil
if stage == "" || l.Stage == stage {
version = l.Version
return nil
}
}
}
@ -63,3 +67,38 @@ func (ds *Store) LatestVersion() (string, error) {
return version, nil
}
// Versions lists the versions in a given project, including the last stage that version got to
func (ds *Store) Versions() ([]*Log, error) {
var vers []*Log
err := ds.bolt.View(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte(bucketLog)).Cursor()
var current = ""
for k, v := c.Last(); k != nil; k, v = c.Prev() {
l := &Log{}
err := json.Unmarshal(v, l)
if err != nil {
return err
}
// capture the newest entry for each version
if l.Version != current {
l.Log = "" // only care about date, ver and stage
vers = append(vers, l)
current = l.Version
}
}
return nil
})
if err != nil {
return nil, err
}
return vers, nil
}

3
main.go
View File

@ -37,7 +37,7 @@ func init() {
go func() {
for sig := range c {
if sig == os.Interrupt {
projects.closeAll()
projects.stopAll()
os.Exit(0)
}
}
@ -91,7 +91,6 @@ func main() {
}
//start web server
err = startServer()
if err != nil {
log.Fatalf("Error Starting web server: %s", err)

238
project.go
View File

@ -5,10 +5,13 @@
package main
import (
"crypto/sha1"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
@ -51,8 +54,8 @@ type Project struct {
Version string `json:"version"` //Script to generate the version num of the current build, should be indempotent
ReleaseFile string `json:"releaseFile"`
PollInterval string `json:"pollInterval"` // if not poll interval is specified, this project is trigger only
TriggerSecret string `json:"triggerSecret"` //secret to be included with a trigger call
PollInterval string `json:"pollInterval,omitempty"` // if not poll interval is specified, this project is trigger only
TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call
filename string
poll time.Duration
@ -60,6 +63,195 @@ type Project struct {
stage string
version string
hash string
sync.RWMutex
}
func (p *Project) errHandled(err error) bool {
if err == nil {
return false
}
vlog("Error in project %s: %s\n", p.id(), err)
if p.ds == nil {
log.Printf("Error in project %s: %s\n", p.id(), err)
return true
}
defer func() {
//clean up version folder if it exists
if p.version != "" {
err = os.RemoveAll(p.workingDir())
if err != nil {
log.Printf("Error deleting the version directory project %s version %s: %s\n",
p.id(), p.version, err)
}
}
}()
lerr := p.ds.AddLog(p.version, p.stage, err.Error())
if lerr != nil {
log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s",
p.id(), err, lerr)
}
return true
}
func projectID(filename string) string {
return strings.TrimSuffix(filename, filepath.Ext(filename))
}
func (p *Project) id() string {
if p.filename == "" {
panic("invalid project filename")
}
return projectID(p.filename)
}
func (p *Project) dir() string {
return filepath.Join(dataDir, p.id())
}
func (p *Project) workingDir() string {
if p.hash == "" {
panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
}
//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
// timestamp on instead would work just fine, but I like having the working dir tied directly to the
// version returned by project script
return filepath.Join(p.dir(), p.hash)
}
// prepData makes sure the project's data folder and data store is created
/*
folder structure
projectDataFolder/<project-name>/<project-version>
*/
func (p *Project) open() error {
p.Lock()
defer p.Unlock()
if p.ds != nil {
return nil
}
ds, err := datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith"))
if err != nil {
return err
}
p.ds = ds
return nil
}
func (p *Project) setVersion(version string) {
p.Lock()
defer p.Unlock()
p.version = version
if version == "" {
p.hash = ""
return
}
p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(version)))
}
func (p *Project) setStage(stage string) {
p.Lock()
defer p.Unlock()
if p.version != "" {
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
} else {
vlog("Entering %s stage for Project: %s\n", p.stage, p.id())
}
p.stage = stage
}
type webProject struct {
ID string `json:"id"`
Name string `json:"name"`
ReleaseVersion string `json:"releaseVersion"` //last successfully released version
LastVersion string `json:"lastVersion"` //last version success or otherwise
}
func (p *Project) webData() (*webProject, error) {
p.RLock()
defer p.RUnlock()
last, err := p.ds.LastVersion("")
if err != nil {
return nil, err
}
release, err := p.ds.LastVersion(stageRelease)
if err != nil {
return nil, err
}
d := &webProject{
Name: p.Name,
ID: p.id(),
LastVersion: last,
ReleaseVersion: release,
}
return d, nil
}
func (p *Project) versions() ([]*datastore.Log, error) {
p.RLock()
defer p.RUnlock()
return p.ds.Versions()
}
func (p *Project) setData(new *Project) {
p.Lock()
defer p.Unlock()
p.Name = new.Name
p.Fetch = new.Fetch
p.Build = new.Build
p.Test = new.Test
p.Release = new.Release
p.Version = new.Version
p.ReleaseFile = new.ReleaseFile
p.PollInterval = new.PollInterval
p.TriggerSecret = new.TriggerSecret
if p.PollInterval != "" {
var err error
p.poll, err = time.ParseDuration(p.PollInterval)
if p.errHandled(err) {
p.poll = 0
}
}
}
func (p *Project) close() error {
p.Lock()
defer p.Unlock()
if p.ds == nil {
return nil
}
err := p.ds.Close()
if err != nil {
return err
}
p.ds = nil
return nil
}
const projectTemplateFilename = "template.project.json"
@ -147,12 +339,12 @@ func (p *projectList) load() error {
return nil
}
func (p *projectList) exists(name string) bool {
func (p *projectList) get(name string) (*Project, bool) {
p.RLock()
defer p.RUnlock()
_, ok := p.data[name]
return ok
prj, ok := p.data[name]
return prj, ok
}
func (p *projectList) add(name string) {
@ -165,9 +357,14 @@ func (p *projectList) add(name string) {
Name: name,
stage: stageLoad,
}
p.data[name] = prj
p.data[projectID(name)] = prj
go func() {
err := prj.open()
if err != nil {
log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err)
return
}
prj.load()
}()
}
@ -180,7 +377,7 @@ func (p *projectList) removeMissing(names []string) {
for i := range p.data {
found := false
for k := range names {
if names[k] == i {
if projectID(names[k]) == i {
found = true
}
}
@ -192,17 +389,36 @@ func (p *projectList) removeMissing(names []string) {
}
}
func (p *projectList) closeAll() {
func (p *projectList) stopAll() {
p.RLock()
defer p.RUnlock()
for i := range p.data {
if p.data[i].ds != nil {
_ = p.data[i].ds.Close()
err := p.data[i].close()
if err != nil {
log.Printf("Error closing project datastore for Project: %s Error: %s\n", p.data[i].id(), err)
}
}
}
func (p *projectList) webList() ([]*webProject, error) {
p.RLock()
defer p.RUnlock()
list := make([]*webProject, 0, len(p.data))
for i := range p.data {
prj, err := p.data[i].webData()
if err != nil {
return nil, err
}
list = append(list, prj)
}
return list, nil
}
// startProjectLoader polls for new projects
func startProjectLoader() {
dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
@ -228,7 +444,7 @@ func startProjectLoader() {
for i := range files {
if !files[i].IsDir() && filepath.Ext(files[i].Name()) == ".json" {
names[i] = files[i].Name()
if !projects.exists(files[i].Name()) {
if _, ok := projects.get(projectID(files[i].Name())); !ok {
projects.add(files[i].Name())
}
}

2
server.go
View File

@ -82,7 +82,7 @@ Routes
/project/ - list all projects
/project/<project-id> - list all versions in a project, triggers new builds
/project/<project-id>/<version> - list combined output of all stages
/project/<project-id>/<version> - list combined output of all stages for a given version
/project/<project-id>/<version>/<stage. - list output of a given stage of a given version
*/

45
webProject.go
View File

@ -5,13 +5,12 @@
package main
import (
"fmt"
"net/http"
"net/url"
"strings"
)
// /project/<project-id>/<version>/<stage>
// /path/<project-id>/<version>/<stage>
func splitPath(path string) (project, version, stage string) {
s := strings.Split(path, "/")
if len(s) < 3 {
@ -33,8 +32,46 @@ func splitPath(path string) (project, version, stage string) {
return
}
// /project/*
func projectGet(w http.ResponseWriter, r *http.Request) {
prj, ver, stg := splitPath(r.URL.Path)
prj, ver, _ := splitPath(r.URL.Path)
//values := r.URL.Query()
if prj == "" {
//get all projects
pList, err := projects.webList()
if errHandled(err, w) {
return
}
respondJsend(w, &JSend{
Status: statusSuccess,
Data: pList,
})
return
}
project, ok := projects.get(prj)
if !ok {
four04(w, r)
return
}
//project found
if ver == "" {
//list versions
vers, err := project.versions()
if errHandled(err, w) {
return
}
respondJsend(w, &JSend{
Status: statusSuccess,
Data: vers,
})
return
}
fmt.Printf("Project: %s Version: %s Stage: %s\n", prj, ver, stg)
}