From 65c489c9208047d67126716601d6259d97297b9f Mon Sep 17 00:00:00 2001 From: Tim Shannon Date: Tue, 5 Apr 2016 21:59:52 +0000 Subject: [PATCH] Added logging and proper execution of commands Fixed several issues, and ran a basic test on ironsmith itself. Need to prevent subsequent builds on same versions --- cycle.go | 119 +++++++++++++++++++++++++++++------------------ datastore/ds.go | 2 +- datastore/log.go | 2 +- exec.go | 33 +++++++++++++ log.go | 13 ++++++ main.go | 45 ++++++++++++++++-- project.go | 27 ++++++++--- server.go | 77 ++++++++++++++++++++++++++++++ 8 files changed, 259 insertions(+), 59 deletions(-) create mode 100644 exec.go create mode 100644 log.go create mode 100644 server.go diff --git a/cycle.go b/cycle.go index 61fd813..3fc61b2 100644 --- a/cycle.go +++ b/cycle.go @@ -5,13 +5,15 @@ package main import ( + "crypto/sha1" "encoding/json" "errors" + "fmt" "io/ioutil" "log" "os" - "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -23,6 +25,8 @@ func (p *Project) errHandled(err error) bool { return false } + vlog("Error in project %s: %s\n", p.id(), err) + if p.ds == nil { log.Printf("Error in project %s: %s\n", p.id(), err) return true @@ -37,9 +41,11 @@ func (p *Project) errHandled(err error) bool { //clean up version folder if it exists if p.version != "" { - err = os.RemoveAll(p.verDir()) - log.Printf("Error deleting the version directory project %s version %s: %s\n", - p.id(), p.version, err) + err = os.RemoveAll(p.workingDir()) + if err != nil { + log.Printf("Error deleting the version directory project %s version %s: %s\n", + p.id(), p.version, err) + } } }() @@ -64,8 +70,16 @@ func (p *Project) dir() string { return filepath.Join(dataDir, p.id()) } -func (p *Project) verDir() string { - return filepath.Join(p.dir(), p.version) +func (p *Project) workingDir() string { + if p.hash == "" { + panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id())) + } + + //It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple + // timestamp on instead would work just fine, but I like having the working dir tied directly to the + // version returned by project script + + return filepath.Join(p.dir(), p.hash) } // prepData makes sure the project's data folder and data store is created @@ -100,6 +114,9 @@ Project life cycle: // call's fetch and triggers the next poll if one exists func (p *Project) load() { p.version = "" + p.hash = "" + + vlog("Entering %s stage for Project: %s\n", stageLoad, p.id()) if p.filename == "" { p.errHandled(errors.New("Invalid project file name")) @@ -109,8 +126,9 @@ func (p *Project) load() { if !projects.exists(p.filename) { // project has been deleted // don't continue polling - // move project data to deleted folder - p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, p.id()))) + // move project data to deleted folder with a timestamp + p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, + strconv.FormatInt(time.Now().Unix(), 10), p.id()))) return } @@ -146,23 +164,33 @@ func (p *Project) load() { } } -// fetch first runs the version script and checks the returned version against the latest version in the -// project database. If the version hasn't changed, then it breaks out of the cycle early doing nothing -// if the version has changed, then it runs the fetch script +// fetch first runs the fetch script into a temporary directory +// then it runs the version script in the temp directory to see if there is a newer version of the +// fetched code, if there is then the temp dir is renamed to the version name func (p *Project) fetch() { p.stage = stageFetch - verCmd := &exec.Cmd{ - Path: p.Version, - Dir: p.dir(), + + vlog("Entering %s stage for Project: %s\n", p.stage, p.id()) + tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10)) + + if p.errHandled(os.MkdirAll(tempDir, 0777)) { + return } - version, err := verCmd.Output() + //fetch project + fetchResult, err := runCmd(p.Fetch, tempDir) + if p.errHandled(err) { + return + } + + // fetched succesfully, determine version + version, err := runCmd(p.Version, tempDir) if p.errHandled(err) { return } - p.version = string(version) + p.version = strings.TrimSpace(string(version)) lVer, err := p.ds.LatestVersion() if err != datastore.ErrNotFound && p.errHandled(err) { @@ -170,30 +198,33 @@ func (p *Project) fetch() { } if p.version == lVer { - // no new build + // no new build clean up temp dir + p.errHandled(os.RemoveAll(tempDir)) + + vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version) return } - if p.errHandled(os.MkdirAll(p.verDir(), 0777)) { + p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(p.version))) + + //remove any existing data that matches version hash + if p.errHandled(os.RemoveAll(p.workingDir())) { return } - //fetch project - fetchCmd := &exec.Cmd{ - Path: p.Fetch, - Dir: p.verDir(), - } - - fetchResult, err := fetchCmd.Output() - if p.errHandled(err) { + //new version move tempdir to workingDir + if p.errHandled(os.Rename(tempDir, p.workingDir())) { + // cleanup temp dir if rename failed + p.errHandled(os.RemoveAll(tempDir)) return } + //log fetch results if p.errHandled(p.ds.AddLog(p.stage, p.version, string(fetchResult))) { return } - // fetched succesfully, onto the build stage + // continue to build p.build() } @@ -202,13 +233,9 @@ func (p *Project) fetch() { // configured in the ReleaseFile section of the project file func (p *Project) build() { p.stage = stageBuild + vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) - buildCmd := &exec.Cmd{ - Path: p.Build, - Dir: p.verDir(), - } - - output, err := buildCmd.Output() + output, err := runCmd(p.Build, p.workingDir()) if p.errHandled(err) { return @@ -225,13 +252,9 @@ func (p *Project) build() { // test runs the test scripts func (p *Project) test() { p.stage = stageTest + vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) - testCmd := &exec.Cmd{ - Path: p.Test, - Dir: p.verDir(), - } - - output, err := testCmd.Output() + output, err := runCmd(p.Test, p.workingDir()) if p.errHandled(err) { return @@ -249,13 +272,9 @@ func (p *Project) test() { // release runs the release scripts and builds the release file func (p *Project) release() { p.stage = stageRelease + vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version) - releaseCmd := &exec.Cmd{ - Path: p.Release, - Dir: p.verDir(), - } - - output, err := releaseCmd.Output() + output, err := runCmd(p.Release, p.workingDir()) if p.errHandled(err) { return @@ -266,7 +285,7 @@ func (p *Project) release() { } //get release file - f, err := os.Open(filepath.Join(p.verDir(), p.ReleaseFile)) + f, err := os.Open(filepath.Join(p.workingDir(), p.ReleaseFile)) if p.errHandled(err) { return } @@ -280,4 +299,12 @@ func (p *Project) release() { return } + //build successfull, remove working dir + p.errHandled(os.RemoveAll(p.workingDir())) + + if p.errHandled(p.ds.Close()) { + return + } + + vlog("Project: %s Version %s built, tested, and released successfully.\n", p.id(), p.version) } diff --git a/datastore/ds.go b/datastore/ds.go index 1f5a85a..5df327a 100644 --- a/datastore/ds.go +++ b/datastore/ds.go @@ -64,7 +64,7 @@ func Open(filename string) (*Store, error) { // Close closes the bolt datastore func (ds *Store) Close() error { - return ds.Close() + return ds.bolt.Close() } func (ds *Store) get(bucket string, key TimeKey, result interface{}) error { diff --git a/datastore/log.go b/datastore/log.go index 0f577ac..8be952e 100644 --- a/datastore/log.go +++ b/datastore/log.go @@ -34,7 +34,7 @@ func (ds *Store) AddLog(version, stage, entry string) error { return ds.put(bucketLog, key, data) } -// LatestVersion returns the latest version for the current project +// LatestVersion returns the latest version (successful or otherwise) for the current project func (ds *Store) LatestVersion() (string, error) { version := "" diff --git a/exec.go b/exec.go new file mode 100644 index 0000000..ceb9e7b --- /dev/null +++ b/exec.go @@ -0,0 +1,33 @@ +// Copyright 2016 Tim Shannon. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "os/exec" + "strings" +) + +func runCmd(cmd, dir string) ([]byte, error) { + s := strings.Fields(cmd) + + var args []string + + if len(s) > 1 { + args = s[1:] + } + + ec := exec.Command(s[0], args...) + + ec.Dir = dir + + vlog("Executing command: %s in dir %s\n", cmd, dir) + + result, err := ec.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("%s:\n%s", err, result) + } + return result, nil +} diff --git a/log.go b/log.go new file mode 100644 index 0000000..2885c3f --- /dev/null +++ b/log.go @@ -0,0 +1,13 @@ +// Copyright 2016 Tim Shannon. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package main + +import "log" + +func vlog(format string, v ...interface{}) { + if verbose { + log.Printf(format, v...) + } +} diff --git a/main.go b/main.go index aeec536..0ea9ff8 100644 --- a/main.go +++ b/main.go @@ -5,8 +5,10 @@ package main import ( + "flag" "log" "os" + "os/signal" "path/filepath" "git.townsourced.com/config" @@ -16,23 +18,46 @@ import ( var ( projectDir = "./projects" // /etc/ironsmith/ dataDir = "./data" // /var/ironsmith/ - address = "http://localhost:8026" + address = ":8026" certFile = "" keyFile = "" ) +//flags +var ( + verbose = false +) + +func init() { + flag.BoolVar(&verbose, "v", false, "Verbose prints to stdOut every command and stage as it processes") + + //Capture program shutdown, to make sure everything shuts down nicely + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + for sig := range c { + if sig == os.Interrupt { + projects.closeAll() + os.Exit(0) + } + } + }() +} + func main() { + flag.Parse() + settingPaths := config.StandardFileLocations("ironsmith/settings.json") - log.Println("IronSmith will use settings files in the following locations (in order of priority):") + vlog("IronSmith will use settings files in the following locations (in order of priority):\n") for i := range settingPaths { - log.Println("\t", settingPaths[i]) + vlog("\t%s\n", settingPaths[i]) } cfg, err := config.LoadOrCreate(settingPaths...) if err != nil { log.Fatalf("Error loading or creating IronSmith settings file: %s", err) } - log.Printf("IronSmith is currently using the file %s for settings.\n", cfg.FileName()) + vlog("IronSmith is currently using the file %s for settings.\n", cfg.FileName()) projectDir = cfg.String("projectDir", projectDir) dataDir = cfg.String("dataDir", dataDir) @@ -40,6 +65,9 @@ func main() { certFile = cfg.String("certFile", certFile) keyFile = cfg.String("keyFile", keyFile) + vlog("Project Definition Directory: %s\n", projectDir) + vlog("Project Data Directory: %s\n", dataDir) + //prep dirs err = os.MkdirAll(filepath.Join(projectDir, enabledProjectDir), 0777) if err != nil { @@ -61,5 +89,12 @@ func main() { if err != nil { log.Fatalf("Error loading projects: %s", err) } - //start server + + //start web server + + err = startServer() + if err != nil { + log.Fatalf("Error Starting web server: %s", err) + } + } diff --git a/project.go b/project.go index 2d5f217..41c4782 100644 --- a/project.go +++ b/project.go @@ -59,6 +59,7 @@ type Project struct { ds *datastore.Store stage string version string + hash string } const projectTemplateFilename = "template.project.json" @@ -66,12 +67,12 @@ const projectTemplateFilename = "template.project.json" var projectTemplate = &Project{ Name: "Template Project", Fetch: "git clone root@git.townsourced.com:tshannon/ironsmith.git .", - Build: "sh ./ironsmith/build.sh", - Test: "sh ./ironsmith/test.sh", - Release: "sh ./ironsmith/release.sh", + Build: "go build -o ironsmith", + Test: "go test ./...", + Release: "tar -czf release.tar.gz ironsmith", Version: "git describe --tags --long", - ReleaseFile: `json:"./ironsmith/release.tar.gz"`, + ReleaseFile: "release.tar.gz", PollInterval: "15m", } @@ -79,6 +80,7 @@ func prepTemplateProject() error { filename := filepath.Join(projectDir, projectTemplateFilename) _, err := os.Stat(filename) if os.IsNotExist(err) { + vlog("Creating template project file in %s", filename) f, err := os.Create(filename) defer func() { if cerr := f.Close(); cerr != nil && err == nil { @@ -117,6 +119,7 @@ var projects = projectList{ } func (p *projectList) load() error { + vlog("Loading projects from the enabled definitions in %s\n", filepath.Join(projectDir, enabledProjectDir)) dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir)) defer func() { if cerr := dir.Close(); cerr != nil && err == nil { @@ -153,6 +156,7 @@ func (p *projectList) exists(name string) bool { } func (p *projectList) add(name string) { + vlog("Adding project %s to the project list.\n", name) p.Lock() defer p.Unlock() @@ -181,11 +185,22 @@ func (p *projectList) removeMissing(names []string) { } } if !found { + vlog("Removing project %s from the project list, because the project file was removed.\n", + i) delete(p.data, i) } } } +func (p *projectList) closeAll() { + p.RLock() + defer p.RUnlock() + + for i := range p.data { + _ = p.data[i].ds.Close() + } +} + // startProjectLoader polls for new projects func startProjectLoader() { dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir)) @@ -196,13 +211,13 @@ func startProjectLoader() { }() if err != nil { - log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", dir, err) + log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", projectDir, err) return } files, err := dir.Readdir(0) if err != nil { - log.Printf("Error in startProjectLoader reading the dir %s: %s\n", dir, err) + log.Printf("Error in startProjectLoader reading the dir %s: %s\n", projectDir, err) return } diff --git a/server.go b/server.go new file mode 100644 index 0000000..3eab399 --- /dev/null +++ b/server.go @@ -0,0 +1,77 @@ +// Copyright 2016 Tim Shannon. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package main + +import "net/http" + +var webRoot *http.ServeMux + +func startServer() error { + var err error + + routes() + + server := &http.Server{ + Handler: webRoot, + Addr: address, + } + + if certFile == "" || keyFile == "" { + err = server.ListenAndServe() + } else { + server.Addr = address + err = server.ListenAndServeTLS(certFile, keyFile) + } + + if err != nil { + return err + } + + return nil +} + +type methodHandler struct { + get http.HandlerFunc + post http.HandlerFunc + put http.HandlerFunc + delete http.HandlerFunc +} + +func (m *methodHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if m.get == nil { + m.get = http.NotFound + } + if m.post == nil { + m.post = http.NotFound + } + if m.put == nil { + m.put = http.NotFound + } + if m.delete == nil { + m.delete = http.NotFound + } + switch r.Method { + case "GET": + m.get(w, r) + return + case "POST": + m.post(w, r) + return + case "PUT": + m.put(w, r) + return + case "DELETE": + m.delete(w, r) + return + default: + http.NotFound(w, r) + return + } +} + +func routes() { + webRoot = http.NewServeMux() + +}