From b9c945ed92a43d84afc885ec6fcd120b0c97b7e3 Mon Sep 17 00:00:00 2001 From: Tim Shannon Date: Wed, 6 Apr 2016 21:59:24 +0000 Subject: [PATCH] Fixed several issues, contineued web work Fixed lots of issues with thread saftey and had to rethink some stuff Fixed order issues with timekeys Starting to flesh out the web REST API --- cycle.go | 129 +++---------------------- datastore/key.go | 10 +- datastore/log.go | 63 ++++++++++--- main.go | 3 +- project.go | 238 ++++++++++++++++++++++++++++++++++++++++++++--- server.go | 2 +- webProject.go | 45 ++++++++- 7 files changed, 340 insertions(+), 150 deletions(-) diff --git a/cycle.go b/cycle.go index b1444bd..726fb1d 100644 --- a/cycle.go +++ b/cycle.go @@ -5,12 +5,9 @@ package main import ( - "crypto/sha1" "encoding/json" "errors" - "fmt" "io/ioutil" - "log" "os" "path/filepath" "strconv" @@ -20,83 +17,6 @@ import ( "git.townsourced.com/ironsmith/datastore" ) -func (p *Project) errHandled(err error) bool { - if err == nil { - return false - } - - vlog("Error in project %s: %s\n", p.id(), err) - - if p.ds == nil { - log.Printf("Error in project %s: %s\n", p.id(), err) - return true - } - defer func() { - //clean up version folder if it exists - - if p.version != "" { - err = os.RemoveAll(p.workingDir()) - if err != nil { - log.Printf("Error deleting the version directory project %s version %s: %s\n", - p.id(), p.version, err) - } - - } - }() - - lerr := p.ds.AddLog(p.version, p.stage, err.Error()) - if lerr != nil { - log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s", - p.id(), err, lerr) - } - - return true -} - -func (p *Project) id() string { - if p.filename == "" { - panic("invalid project filename") - } - return strings.TrimSuffix(p.filename, filepath.Ext(p.filename)) -} - -func (p *Project) dir() string { - return filepath.Join(dataDir, p.id()) -} - -func (p *Project) workingDir() string { - if p.hash == "" { - panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id())) - } - - //It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple - // timestamp on instead would work just fine, but I like having the working dir tied directly to the - // version returned by project script - - return filepath.Join(p.dir(), p.hash) -} - -// prepData makes sure the project's data folder and data store is created -/* - folder structure - projectDataFolder// - -*/ -func (p *Project) prepData() error { - err := os.MkdirAll(p.dir(), 0777) - if err != nil { - return err - } - - p.ds, err = datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith")) - - if err != nil { - return err - } - - return nil -} - /* Project life cycle: (Load Project file) -> (Fetch) -> (Build) -> (Test) -> (Release) - > (Sleep for polling period) -> @@ -107,20 +27,19 @@ Project life cycle: // load is the beginning of the cycle. Loads / reloads the project file to make sure that the scripts are up-to-date // call's fetch and triggers the next poll if one exists func (p *Project) load() { - p.version = "" - p.hash = "" - - vlog("Entering %s stage for Project: %s\n", stageLoad, p.id()) + p.setStage(stageLoad) + p.setVersion("") if p.filename == "" { p.errHandled(errors.New("Invalid project file name")) return } - if !projects.exists(p.filename) { + if _, ok := projects.get(p.id()); !ok { // project has been deleted // don't continue polling // move project data to deleted folder with a timestamp + p.close() p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, strconv.FormatInt(time.Now().Unix(), 10), p.id()))) return @@ -131,30 +50,19 @@ func (p *Project) load() { return } - if p.errHandled(json.Unmarshal(data, p)) { + new := &Project{} + if p.errHandled(json.Unmarshal(data, new)) { return } - p.stage = stageLoad + p.setData(new) - if p.errHandled(p.prepData()) { + if p.errHandled(os.MkdirAll(p.dir(), 0777)) { return } - if p.PollInterval != "" { - p.poll, err = time.ParseDuration(p.PollInterval) - if p.errHandled(err) { - p.poll = 0 - } - } - p.fetch() - if p.errHandled(p.ds.Close()) { - return - } - p.ds = nil - //full cycle completed if p.poll > 0 { @@ -167,9 +75,8 @@ func (p *Project) load() { // then it runs the version script in the temp directory to see if there is a newer version of the // fetched code, if there is then the temp dir is renamed to the version name func (p *Project) fetch() { - p.stage = stageFetch + p.setStage(stageFetch) - vlog("Entering %s stage for Project: %s\n", p.stage, p.id()) tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10)) if p.errHandled(os.MkdirAll(tempDir, 0777)) { @@ -189,9 +96,9 @@ func (p *Project) fetch() { return } - p.version = strings.TrimSpace(string(version)) + p.setVersion(strings.TrimSpace(string(version))) - lVer, err := p.ds.LatestVersion() + lVer, err := p.ds.LastVersion("") if err != datastore.ErrNotFound && p.errHandled(err) { return } @@ -204,8 +111,6 @@ func (p *Project) fetch() { return } - p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(p.version))) - //remove any existing data that matches version hash if p.errHandled(os.RemoveAll(p.workingDir())) { return @@ -225,14 +130,12 @@ func (p *Project) fetch() { // continue to build p.build() - } // build runs the build scripts to build the project which should result in the a single file // configured in the ReleaseFile section of the project file func (p *Project) build() { - p.stage = stageBuild - vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) + p.setStage(stageBuild) output, err := runCmd(p.Build, p.workingDir()) @@ -250,9 +153,7 @@ func (p *Project) build() { // test runs the test scripts func (p *Project) test() { - p.stage = stageTest - vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) - + p.setStage(stageTest) output, err := runCmd(p.Test, p.workingDir()) if p.errHandled(err) { @@ -265,13 +166,11 @@ func (p *Project) test() { // Tests passed, onto release p.release() - } // release runs the release scripts and builds the release file func (p *Project) release() { - p.stage = stageRelease - vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) + p.setStage(stageRelease) output, err := runCmd(p.Release, p.workingDir()) diff --git a/datastore/key.go b/datastore/key.go index 4b7517c..b5838e6 100644 --- a/datastore/key.go +++ b/datastore/key.go @@ -29,10 +29,6 @@ func NewTimeKey() TimeKey { nsec := t.Nanosecond() return TimeKey{ - rBits[0], //random - rBits[1], - rBits[2], - rBits[3], byte(sec >> 56), // seconds byte(sec >> 48), byte(sec >> 40), @@ -45,12 +41,16 @@ func NewTimeKey() TimeKey { byte(nsec >> 16), byte(nsec >> 8), byte(nsec), + rBits[0], //random + rBits[1], + rBits[2], + rBits[3], } } // Time returns the time portion of a timekey func (k TimeKey) Time() time.Time { - buf := k[4:] + buf := k[:] sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 | int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56 diff --git a/datastore/log.go b/datastore/log.go index a9ae40c..d40f69a 100644 --- a/datastore/log.go +++ b/datastore/log.go @@ -11,11 +11,12 @@ import ( "github.com/boltdb/bolt" ) -type log struct { - When time.Time `json:"when"` - Version string `json:"version"` - Stage string `json:"stage"` - Log string `json:"log"` +// Log is a version log entry for a project +type Log struct { + When time.Time `json:"when,omitempty"` + Version string `json:"version,omitempty"` + Stage string `json:"stage,omitempty"` + Log string `json:"log,omitempty"` } const bucketLog = "log" @@ -24,7 +25,7 @@ const bucketLog = "log" func (ds *Store) AddLog(version, stage, entry string) error { key := NewTimeKey() - data := &log{ + data := &Log{ When: key.Time(), Version: version, Stage: stage, @@ -34,23 +35,26 @@ func (ds *Store) AddLog(version, stage, entry string) error { return ds.put(bucketLog, key, data) } -// LatestVersion returns the latest version (successful or otherwise) for the current project -func (ds *Store) LatestVersion() (string, error) { +// LastVersion returns the last version in the log for the given stage. If stage is blank, +// then it returns the last of any stage +func (ds *Store) LastVersion(stage string) (string, error) { version := "" err := ds.bolt.View(func(tx *bolt.Tx) error { c := tx.Bucket([]byte(bucketLog)).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - l := &log{} + for k, v := c.Last(); k != nil; k, v = c.Prev() { + l := &Log{} err := json.Unmarshal(v, l) if err != nil { return err } if l.Version != "" { - version = l.Version - return nil + if stage == "" || l.Stage == stage { + version = l.Version + return nil + } } } @@ -63,3 +67,38 @@ func (ds *Store) LatestVersion() (string, error) { return version, nil } + +// Versions lists the versions in a given project, including the last stage that version got to +func (ds *Store) Versions() ([]*Log, error) { + var vers []*Log + + err := ds.bolt.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(bucketLog)).Cursor() + + var current = "" + + for k, v := c.Last(); k != nil; k, v = c.Prev() { + l := &Log{} + err := json.Unmarshal(v, l) + if err != nil { + return err + } + + // capture the newest entry for each version + if l.Version != current { + l.Log = "" // only care about date, ver and stage + vers = append(vers, l) + current = l.Version + } + + } + + return nil + }) + + if err != nil { + return nil, err + } + + return vers, nil +} diff --git a/main.go b/main.go index 0ea9ff8..36c46d1 100644 --- a/main.go +++ b/main.go @@ -37,7 +37,7 @@ func init() { go func() { for sig := range c { if sig == os.Interrupt { - projects.closeAll() + projects.stopAll() os.Exit(0) } } @@ -91,7 +91,6 @@ func main() { } //start web server - err = startServer() if err != nil { log.Fatalf("Error Starting web server: %s", err) diff --git a/project.go b/project.go index ef15906..86841c1 100644 --- a/project.go +++ b/project.go @@ -5,10 +5,13 @@ package main import ( + "crypto/sha1" "encoding/json" + "fmt" "log" "os" "path/filepath" + "strings" "sync" "time" @@ -51,8 +54,8 @@ type Project struct { Version string `json:"version"` //Script to generate the version num of the current build, should be indempotent ReleaseFile string `json:"releaseFile"` - PollInterval string `json:"pollInterval"` // if not poll interval is specified, this project is trigger only - TriggerSecret string `json:"triggerSecret"` //secret to be included with a trigger call + PollInterval string `json:"pollInterval,omitempty"` // if not poll interval is specified, this project is trigger only + TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call filename string poll time.Duration @@ -60,6 +63,195 @@ type Project struct { stage string version string hash string + + sync.RWMutex +} + +func (p *Project) errHandled(err error) bool { + if err == nil { + return false + } + + vlog("Error in project %s: %s\n", p.id(), err) + + if p.ds == nil { + log.Printf("Error in project %s: %s\n", p.id(), err) + return true + } + defer func() { + //clean up version folder if it exists + if p.version != "" { + err = os.RemoveAll(p.workingDir()) + if err != nil { + log.Printf("Error deleting the version directory project %s version %s: %s\n", + p.id(), p.version, err) + } + + } + }() + + lerr := p.ds.AddLog(p.version, p.stage, err.Error()) + if lerr != nil { + log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s", + p.id(), err, lerr) + } + + return true +} + +func projectID(filename string) string { + return strings.TrimSuffix(filename, filepath.Ext(filename)) +} + +func (p *Project) id() string { + if p.filename == "" { + panic("invalid project filename") + } + return projectID(p.filename) +} + +func (p *Project) dir() string { + return filepath.Join(dataDir, p.id()) +} + +func (p *Project) workingDir() string { + if p.hash == "" { + panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id())) + } + + //It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple + // timestamp on instead would work just fine, but I like having the working dir tied directly to the + // version returned by project script + + return filepath.Join(p.dir(), p.hash) +} + +// prepData makes sure the project's data folder and data store is created +/* + folder structure + projectDataFolder// + +*/ +func (p *Project) open() error { + p.Lock() + defer p.Unlock() + + if p.ds != nil { + return nil + } + + ds, err := datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith")) + if err != nil { + return err + } + + p.ds = ds + + return nil +} + +func (p *Project) setVersion(version string) { + p.Lock() + defer p.Unlock() + + p.version = version + if version == "" { + p.hash = "" + return + } + + p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(version))) +} + +func (p *Project) setStage(stage string) { + p.Lock() + defer p.Unlock() + + if p.version != "" { + vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) + } else { + vlog("Entering %s stage for Project: %s\n", p.stage, p.id()) + } + + p.stage = stage +} + +type webProject struct { + ID string `json:"id"` + Name string `json:"name"` + ReleaseVersion string `json:"releaseVersion"` //last successfully released version + LastVersion string `json:"lastVersion"` //last version success or otherwise +} + +func (p *Project) webData() (*webProject, error) { + p.RLock() + defer p.RUnlock() + + last, err := p.ds.LastVersion("") + if err != nil { + return nil, err + } + + release, err := p.ds.LastVersion(stageRelease) + if err != nil { + return nil, err + } + + d := &webProject{ + Name: p.Name, + ID: p.id(), + LastVersion: last, + ReleaseVersion: release, + } + + return d, nil +} + +func (p *Project) versions() ([]*datastore.Log, error) { + p.RLock() + defer p.RUnlock() + + return p.ds.Versions() +} + +func (p *Project) setData(new *Project) { + p.Lock() + defer p.Unlock() + + p.Name = new.Name + + p.Fetch = new.Fetch + p.Build = new.Build + p.Test = new.Test + p.Release = new.Release + p.Version = new.Version + + p.ReleaseFile = new.ReleaseFile + p.PollInterval = new.PollInterval + p.TriggerSecret = new.TriggerSecret + + if p.PollInterval != "" { + var err error + p.poll, err = time.ParseDuration(p.PollInterval) + if p.errHandled(err) { + p.poll = 0 + } + } +} + +func (p *Project) close() error { + p.Lock() + defer p.Unlock() + if p.ds == nil { + return nil + } + err := p.ds.Close() + if err != nil { + return err + } + + p.ds = nil + return nil } const projectTemplateFilename = "template.project.json" @@ -147,12 +339,12 @@ func (p *projectList) load() error { return nil } -func (p *projectList) exists(name string) bool { +func (p *projectList) get(name string) (*Project, bool) { p.RLock() defer p.RUnlock() - _, ok := p.data[name] - return ok + prj, ok := p.data[name] + return prj, ok } func (p *projectList) add(name string) { @@ -165,9 +357,14 @@ func (p *projectList) add(name string) { Name: name, stage: stageLoad, } - p.data[name] = prj + p.data[projectID(name)] = prj go func() { + err := prj.open() + if err != nil { + log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err) + return + } prj.load() }() } @@ -180,7 +377,7 @@ func (p *projectList) removeMissing(names []string) { for i := range p.data { found := false for k := range names { - if names[k] == i { + if projectID(names[k]) == i { found = true } } @@ -192,17 +389,36 @@ func (p *projectList) removeMissing(names []string) { } } -func (p *projectList) closeAll() { +func (p *projectList) stopAll() { p.RLock() defer p.RUnlock() for i := range p.data { - if p.data[i].ds != nil { - _ = p.data[i].ds.Close() + err := p.data[i].close() + if err != nil { + log.Printf("Error closing project datastore for Project: %s Error: %s\n", p.data[i].id(), err) } } } +func (p *projectList) webList() ([]*webProject, error) { + p.RLock() + defer p.RUnlock() + + list := make([]*webProject, 0, len(p.data)) + + for i := range p.data { + prj, err := p.data[i].webData() + if err != nil { + return nil, err + } + + list = append(list, prj) + } + + return list, nil +} + // startProjectLoader polls for new projects func startProjectLoader() { dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir)) @@ -228,7 +444,7 @@ func startProjectLoader() { for i := range files { if !files[i].IsDir() && filepath.Ext(files[i].Name()) == ".json" { names[i] = files[i].Name() - if !projects.exists(files[i].Name()) { + if _, ok := projects.get(projectID(files[i].Name())); !ok { projects.add(files[i].Name()) } } diff --git a/server.go b/server.go index 0c70875..c06aa07 100644 --- a/server.go +++ b/server.go @@ -82,7 +82,7 @@ Routes /project/ - list all projects /project/ - list all versions in a project, triggers new builds - /project// - list combined output of all stages + /project// - list combined output of all stages for a given version /project///// +// /path/// func splitPath(path string) (project, version, stage string) { s := strings.Split(path, "/") if len(s) < 3 { @@ -33,8 +32,46 @@ func splitPath(path string) (project, version, stage string) { return } +// /project/* func projectGet(w http.ResponseWriter, r *http.Request) { - prj, ver, stg := splitPath(r.URL.Path) + prj, ver, _ := splitPath(r.URL.Path) + + //values := r.URL.Query() + + if prj == "" { + //get all projects + pList, err := projects.webList() + if errHandled(err, w) { + return + } + + respondJsend(w, &JSend{ + Status: statusSuccess, + Data: pList, + }) + + return + } + + project, ok := projects.get(prj) + if !ok { + four04(w, r) + return + } + + //project found + + if ver == "" { + //list versions + vers, err := project.versions() + if errHandled(err, w) { + return + } + respondJsend(w, &JSend{ + Status: statusSuccess, + Data: vers, + }) + return + } - fmt.Printf("Project: %s Version: %s Stage: %s\n", prj, ver, stg) }