diff --git a/cycle.go b/cycle.go index b1444bd..726fb1d 100644 --- a/cycle.go +++ b/cycle.go @@ -5,12 +5,9 @@ package main import ( - "crypto/sha1" "encoding/json" "errors" - "fmt" "io/ioutil" - "log" "os" "path/filepath" "strconv" @@ -20,83 +17,6 @@ import ( "git.townsourced.com/ironsmith/datastore" ) -func (p *Project) errHandled(err error) bool { - if err == nil { - return false - } - - vlog("Error in project %s: %s\n", p.id(), err) - - if p.ds == nil { - log.Printf("Error in project %s: %s\n", p.id(), err) - return true - } - defer func() { - //clean up version folder if it exists - - if p.version != "" { - err = os.RemoveAll(p.workingDir()) - if err != nil { - log.Printf("Error deleting the version directory project %s version %s: %s\n", - p.id(), p.version, err) - } - - } - }() - - lerr := p.ds.AddLog(p.version, p.stage, err.Error()) - if lerr != nil { - log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s", - p.id(), err, lerr) - } - - return true -} - -func (p *Project) id() string { - if p.filename == "" { - panic("invalid project filename") - } - return strings.TrimSuffix(p.filename, filepath.Ext(p.filename)) -} - -func (p *Project) dir() string { - return filepath.Join(dataDir, p.id()) -} - -func (p *Project) workingDir() string { - if p.hash == "" { - panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id())) - } - - //It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple - // timestamp on instead would work just fine, but I like having the working dir tied directly to the - // version returned by project script - - return filepath.Join(p.dir(), p.hash) -} - -// prepData makes sure the project's data folder and data store is created -/* - folder structure - projectDataFolder// - -*/ -func (p *Project) prepData() error { - err := os.MkdirAll(p.dir(), 0777) - if err != nil { - return err - } - - p.ds, err = datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith")) - - if err != nil { - return err - } - - return nil -} - /* Project life cycle: (Load Project file) -> (Fetch) -> (Build) -> (Test) -> (Release) - > (Sleep for polling period) -> @@ -107,20 +27,19 @@ Project life cycle: // load is the beginning of the cycle. Loads / reloads the project file to make sure that the scripts are up-to-date // call's fetch and triggers the next poll if one exists func (p *Project) load() { - p.version = "" - p.hash = "" - - vlog("Entering %s stage for Project: %s\n", stageLoad, p.id()) + p.setStage(stageLoad) + p.setVersion("") if p.filename == "" { p.errHandled(errors.New("Invalid project file name")) return } - if !projects.exists(p.filename) { + if _, ok := projects.get(p.id()); !ok { // project has been deleted // don't continue polling // move project data to deleted folder with a timestamp + p.close() p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, strconv.FormatInt(time.Now().Unix(), 10), p.id()))) return @@ -131,30 +50,19 @@ func (p *Project) load() { return } - if p.errHandled(json.Unmarshal(data, p)) { + new := &Project{} + if p.errHandled(json.Unmarshal(data, new)) { return } - p.stage = stageLoad + p.setData(new) - if p.errHandled(p.prepData()) { + if p.errHandled(os.MkdirAll(p.dir(), 0777)) { return } - if p.PollInterval != "" { - p.poll, err = time.ParseDuration(p.PollInterval) - if p.errHandled(err) { - p.poll = 0 - } - } - p.fetch() - if p.errHandled(p.ds.Close()) { - return - } - p.ds = nil - //full cycle completed if p.poll > 0 { @@ -167,9 +75,8 @@ func (p *Project) load() { // then it runs the version script in the temp directory to see if there is a newer version of the // fetched code, if there is then the temp dir is renamed to the version name func (p *Project) fetch() { - p.stage = stageFetch + p.setStage(stageFetch) - vlog("Entering %s stage for Project: %s\n", p.stage, p.id()) tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10)) if p.errHandled(os.MkdirAll(tempDir, 0777)) { @@ -189,9 +96,9 @@ func (p *Project) fetch() { return } - p.version = strings.TrimSpace(string(version)) + p.setVersion(strings.TrimSpace(string(version))) - lVer, err := p.ds.LatestVersion() + lVer, err := p.ds.LastVersion("") if err != datastore.ErrNotFound && p.errHandled(err) { return } @@ -204,8 +111,6 @@ func (p *Project) fetch() { return } - p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(p.version))) - //remove any existing data that matches version hash if p.errHandled(os.RemoveAll(p.workingDir())) { return @@ -225,14 +130,12 @@ func (p *Project) fetch() { // continue to build p.build() - } // build runs the build scripts to build the project which should result in the a single file // configured in the ReleaseFile section of the project file func (p *Project) build() { - p.stage = stageBuild - vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) + p.setStage(stageBuild) output, err := runCmd(p.Build, p.workingDir()) @@ -250,9 +153,7 @@ func (p *Project) build() { // test runs the test scripts func (p *Project) test() { - p.stage = stageTest - vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) - + p.setStage(stageTest) output, err := runCmd(p.Test, p.workingDir()) if p.errHandled(err) { @@ -265,13 +166,11 @@ func (p *Project) test() { // Tests passed, onto release p.release() - } // release runs the release scripts and builds the release file func (p *Project) release() { - p.stage = stageRelease - vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) + p.setStage(stageRelease) output, err := runCmd(p.Release, p.workingDir()) diff --git a/datastore/key.go b/datastore/key.go index 4b7517c..b5838e6 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -29,10 +29,6 @@ func NewTimeKey() TimeKey { nsec := t.Nanosecond() return TimeKey{ - rBits[0], //random - rBits[1], - rBits[2], - rBits[3], byte(sec >> 56), // seconds byte(sec >> 48), byte(sec >> 40), @@ -45,12 +41,16 @@ func NewTimeKey() TimeKey { byte(nsec >> 16), byte(nsec >> 8), byte(nsec), + rBits[0], //random + rBits[1], + rBits[2], + rBits[3], } } // Time returns the time portion of a timekey func (k TimeKey) Time() time.Time { - buf := k[4:] + buf := k[:] sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 | int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56 diff --git a/datastore/log.go b/datastore/log.go index a9ae40c..d40f69a 100644 --- a/datastore/log.go +++ b/datastore/log.go @@ -11,11 +11,12 @@ import ( "github.com/boltdb/bolt" ) -type log struct { - When time.Time `json:"when"` - Version string `json:"version"` - Stage string `json:"stage"` - Log string `json:"log"` +// Log is a version log entry for a project +type Log struct { + When time.Time `json:"when,omitempty"` + Version string `json:"version,omitempty"` + Stage string `json:"stage,omitempty"` + Log string `json:"log,omitempty"` } const bucketLog = "log" @@ -24,7 +25,7 @@ const bucketLog = "log" func (ds *Store) AddLog(version, stage, entry string) error { key := NewTimeKey() - data := &log{ + data := &Log{ When: key.Time(), Version: version, Stage: stage, @@ -34,23 +35,26 @@ func (ds *Store) AddLog(version, stage, entry string) error { return ds.put(bucketLog, key, data) } -// LatestVersion returns the latest version (successful or otherwise) for the current project -func (ds *Store) LatestVersion() (string, error) { +// LastVersion returns the last version in the log for the given stage. If stage is blank, +// then it returns the last of any stage +func (ds *Store) LastVersion(stage string) (string, error) { version := "" err := ds.bolt.View(func(tx *bolt.Tx) error { c := tx.Bucket([]byte(bucketLog)).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - l := &log{} + for k, v := c.Last(); k != nil; k, v = c.Prev() { + l := &Log{} err := json.Unmarshal(v, l) if err != nil { return err } if l.Version != "" { - version = l.Version - return nil + if stage == "" || l.Stage == stage { + version = l.Version + return nil + } } } @@ -63,3 +67,38 @@ func (ds *Store) LatestVersion() (string, error) { return version, nil } + +// Versions lists the versions in a given project, including the last stage that version got to +func (ds *Store) Versions() ([]*Log, error) { + var vers []*Log + + err := ds.bolt.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(bucketLog)).Cursor() + + var current = "" + + for k, v := c.Last(); k != nil; k, v = c.Prev() { + l := &Log{} + err := json.Unmarshal(v, l) + if err != nil { + return err + } + + // capture the newest entry for each version + if l.Version != current { + l.Log = "" // only care about date, ver and stage + vers = append(vers, l) + current = l.Version + } + + } + + return nil + }) + + if err != nil { + return nil, err + } + + return vers, nil +} diff --git a/main.go b/main.go index 0ea9ff8..36c46d1 100644 --- a/main.go +++ b/main.go @@ -37,7 +37,7 @@ func init() { go func() { for sig := range c { if sig == os.Interrupt { - projects.closeAll() + projects.stopAll() os.Exit(0) } } @@ -91,7 +91,6 @@ func main() { } //start web server - err = startServer() if err != nil { log.Fatalf("Error Starting web server: %s", err) diff --git a/project.go b/project.go index ef15906..86841c1 100644 --- a/project.go +++ b/project.go @@ -5,10 +5,13 @@ package main import ( + "crypto/sha1" "encoding/json" + "fmt" "log" "os" "path/filepath" + "strings" "sync" "time" @@ -51,8 +54,8 @@ type Project struct { Version string `json:"version"` //Script to generate the version num of the current build, should be indempotent ReleaseFile string `json:"releaseFile"` - PollInterval string `json:"pollInterval"` // if not poll interval is specified, this project is trigger only - TriggerSecret string `json:"triggerSecret"` //secret to be included with a trigger call + PollInterval string `json:"pollInterval,omitempty"` // if not poll interval is specified, this project is trigger only + TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call filename string poll time.Duration @@ -60,6 +63,195 @@ type Project struct { stage string version string hash string + + sync.RWMutex +} + +func (p *Project) errHandled(err error) bool { + if err == nil { + return false + } + + vlog("Error in project %s: %s\n", p.id(), err) + + if p.ds == nil { + log.Printf("Error in project %s: %s\n", p.id(), err) + return true + } + defer func() { + //clean up version folder if it exists + if p.version != "" { + err = os.RemoveAll(p.workingDir()) + if err != nil { + log.Printf("Error deleting the version directory project %s version %s: %s\n", + p.id(), p.version, err) + } + + } + }() + + lerr := p.ds.AddLog(p.version, p.stage, err.Error()) + if lerr != nil { + log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s", + p.id(), err, lerr) + } + + return true +} + +func projectID(filename string) string { + return strings.TrimSuffix(filename, filepath.Ext(filename)) +} + +func (p *Project) id() string { + if p.filename == "" { + panic("invalid project filename") + } + return projectID(p.filename) +} + +func (p *Project) dir() string { + return filepath.Join(dataDir, p.id()) +} + +func (p *Project) workingDir() string { + if p.hash == "" { + panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id())) + } + + //It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple + // timestamp on instead would work just fine, but I like having the working dir tied directly to the + // version returned by project script + + return filepath.Join(p.dir(), p.hash) +} + +// prepData makes sure the project's data folder and data store is created +/* + folder structure + projectDataFolder// + +*/ +func (p *Project) open() error { + p.Lock() + defer p.Unlock() + + if p.ds != nil { + return nil + } + + ds, err := datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith")) + if err != nil { + return err + } + + p.ds = ds + + return nil +} + +func (p *Project) setVersion(version string) { + p.Lock() + defer p.Unlock() + + p.version = version + if version == "" { + p.hash = "" + return + } + + p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(version))) +} + +func (p *Project) setStage(stage string) { + p.Lock() + defer p.Unlock() + + if p.version != "" { + vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) + } else { + vlog("Entering %s stage for Project: %s\n", p.stage, p.id()) + } + + p.stage = stage +} + +type webProject struct { + ID string `json:"id"` + Name string `json:"name"` + ReleaseVersion string `json:"releaseVersion"` //last successfully released version + LastVersion string `json:"lastVersion"` //last version success or otherwise +} + +func (p *Project) webData() (*webProject, error) { + p.RLock() + defer p.RUnlock() + + last, err := p.ds.LastVersion("") + if err != nil { + return nil, err + } + + release, err := p.ds.LastVersion(stageRelease) + if err != nil { + return nil, err + } + + d := &webProject{ + Name: p.Name, + ID: p.id(), + LastVersion: last, + ReleaseVersion: release, + } + + return d, nil +} + +func (p *Project) versions() ([]*datastore.Log, error) { + p.RLock() + defer p.RUnlock() + + return p.ds.Versions() +} + +func (p *Project) setData(new *Project) { + p.Lock() + defer p.Unlock() + + p.Name = new.Name + + p.Fetch = new.Fetch + p.Build = new.Build + p.Test = new.Test + p.Release = new.Release + p.Version = new.Version + + p.ReleaseFile = new.ReleaseFile + p.PollInterval = new.PollInterval + p.TriggerSecret = new.TriggerSecret + + if p.PollInterval != "" { + var err error + p.poll, err = time.ParseDuration(p.PollInterval) + if p.errHandled(err) { + p.poll = 0 + } + } +} + +func (p *Project) close() error { + p.Lock() + defer p.Unlock() + if p.ds == nil { + return nil + } + err := p.ds.Close() + if err != nil { + return err + } + + p.ds = nil + return nil } const projectTemplateFilename = "template.project.json" @@ -147,12 +339,12 @@ func (p *projectList) load() error { return nil } -func (p *projectList) exists(name string) bool { +func (p *projectList) get(name string) (*Project, bool) { p.RLock() defer p.RUnlock() - _, ok := p.data[name] - return ok + prj, ok := p.data[name] + return prj, ok } func (p *projectList) add(name string) { @@ -165,9 +357,14 @@ func (p *projectList) add(name string) { Name: name, stage: stageLoad, } - p.data[name] = prj + p.data[projectID(name)] = prj go func() { + err := prj.open() + if err != nil { + log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err) + return + } prj.load() }() } @@ -180,7 +377,7 @@ func (p *projectList) removeMissing(names []string) { for i := range p.data { found := false for k := range names { - if names[k] == i { + if projectID(names[k]) == i { found = true } } @@ -192,17 +389,36 @@ func (p *projectList) removeMissing(names []string) { } } -func (p *projectList) closeAll() { +func (p *projectList) stopAll() { p.RLock() defer p.RUnlock() for i := range p.data { - if p.data[i].ds != nil { - _ = p.data[i].ds.Close() + err := p.data[i].close() + if err != nil { + log.Printf("Error closing project datastore for Project: %s Error: %s\n", p.data[i].id(), err) } } } +func (p *projectList) webList() ([]*webProject, error) { + p.RLock() + defer p.RUnlock() + + list := make([]*webProject, 0, len(p.data)) + + for i := range p.data { + prj, err := p.data[i].webData() + if err != nil { + return nil, err + } + + list = append(list, prj) + } + + return list, nil +} + // startProjectLoader polls for new projects func startProjectLoader() { dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir)) @@ -228,7 +444,7 @@ func startProjectLoader() { for i := range files { if !files[i].IsDir() && filepath.Ext(files[i].Name()) == ".json" { names[i] = files[i].Name() - if !projects.exists(files[i].Name()) { + if _, ok := projects.get(projectID(files[i].Name())); !ok { projects.add(files[i].Name()) } } diff --git a/server.go b/server.go index 0c70875..c06aa07 100644 --- a/server.go +++ b/server.go @@ -82,7 +82,7 @@ Routes /project/ - list all projects /project/ - list all versions in a project, triggers new builds - /project// - list combined output of all stages + /project// - list combined output of all stages for a given version /project///// +// /path/// func splitPath(path string) (project, version, stage string) { s := strings.Split(path, "/") if len(s) < 3 { @@ -33,8 +32,46 @@ func splitPath(path string) (project, version, stage string) { return } +// /project/* func projectGet(w http.ResponseWriter, r *http.Request) { - prj, ver, stg := splitPath(r.URL.Path) + prj, ver, _ := splitPath(r.URL.Path) + + //values := r.URL.Query() + + if prj == "" { + //get all projects + pList, err := projects.webList() + if errHandled(err, w) { + return + } + + respondJsend(w, &JSend{ + Status: statusSuccess, + Data: pList, + }) + + return + } + + project, ok := projects.get(prj) + if !ok { + four04(w, r) + return + } + + //project found + + if ver == "" { + //list versions + vers, err := project.versions() + if errHandled(err, w) { + return + } + respondJsend(w, &JSend{ + Status: statusSuccess, + Data: vers, + }) + return + } - fmt.Printf("Project: %s Version: %s Stage: %s\n", prj, ver, stg) }