Added logging and proper execution of commands

Fixed several issues, and ran a basic test on ironsmith itself.

Need to prevent subsequent builds on same versions
This commit is contained in:
Tim Shannon 2016-04-05 21:59:52 +00:00
parent 38d20d46fe
commit 65c489c920
8 changed files with 259 additions and 59 deletions

115
cycle.go
View File

@ -5,13 +5,15 @@
package main package main
import ( import (
"crypto/sha1"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"io/ioutil" "io/ioutil"
"log" "log"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"strconv"
"strings" "strings"
"time" "time"
@ -23,6 +25,8 @@ func (p *Project) errHandled(err error) bool {
return false return false
} }
vlog("Error in project %s: %s\n", p.id(), err)
if p.ds == nil { if p.ds == nil {
log.Printf("Error in project %s: %s\n", p.id(), err) log.Printf("Error in project %s: %s\n", p.id(), err)
return true return true
@ -37,9 +41,11 @@ func (p *Project) errHandled(err error) bool {
//clean up version folder if it exists //clean up version folder if it exists
if p.version != "" { if p.version != "" {
err = os.RemoveAll(p.verDir()) err = os.RemoveAll(p.workingDir())
if err != nil {
log.Printf("Error deleting the version directory project %s version %s: %s\n", log.Printf("Error deleting the version directory project %s version %s: %s\n",
p.id(), p.version, err) p.id(), p.version, err)
}
} }
}() }()
@ -64,8 +70,16 @@ func (p *Project) dir() string {
return filepath.Join(dataDir, p.id()) return filepath.Join(dataDir, p.id())
} }
func (p *Project) verDir() string { func (p *Project) workingDir() string {
return filepath.Join(p.dir(), p.version) if p.hash == "" {
panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
}
//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
// timestamp on instead would work just fine, but I like having the working dir tied directly to the
// version returned by project script
return filepath.Join(p.dir(), p.hash)
} }
// prepData makes sure the project's data folder and data store is created // prepData makes sure the project's data folder and data store is created
@ -100,6 +114,9 @@ Project life cycle:
// call's fetch and triggers the next poll if one exists // call's fetch and triggers the next poll if one exists
func (p *Project) load() { func (p *Project) load() {
p.version = "" p.version = ""
p.hash = ""
vlog("Entering %s stage for Project: %s\n", stageLoad, p.id())
if p.filename == "" { if p.filename == "" {
p.errHandled(errors.New("Invalid project file name")) p.errHandled(errors.New("Invalid project file name"))
@ -109,8 +126,9 @@ func (p *Project) load() {
if !projects.exists(p.filename) { if !projects.exists(p.filename) {
// project has been deleted // project has been deleted
// don't continue polling // don't continue polling
// move project data to deleted folder // move project data to deleted folder with a timestamp
p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, p.id()))) p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir,
strconv.FormatInt(time.Now().Unix(), 10), p.id())))
return return
} }
@ -146,23 +164,33 @@ func (p *Project) load() {
} }
} }
// fetch first runs the version script and checks the returned version against the latest version in the // fetch first runs the fetch script into a temporary directory
// project database. If the version hasn't changed, then it breaks out of the cycle early doing nothing // then it runs the version script in the temp directory to see if there is a newer version of the
// if the version has changed, then it runs the fetch script // fetched code, if there is then the temp dir is renamed to the version name
func (p *Project) fetch() { func (p *Project) fetch() {
p.stage = stageFetch p.stage = stageFetch
verCmd := &exec.Cmd{
Path: p.Version, vlog("Entering %s stage for Project: %s\n", p.stage, p.id())
Dir: p.dir(), tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10))
if p.errHandled(os.MkdirAll(tempDir, 0777)) {
return
} }
version, err := verCmd.Output() //fetch project
fetchResult, err := runCmd(p.Fetch, tempDir)
if p.errHandled(err) {
return
}
// fetched succesfully, determine version
version, err := runCmd(p.Version, tempDir)
if p.errHandled(err) { if p.errHandled(err) {
return return
} }
p.version = string(version) p.version = strings.TrimSpace(string(version))
lVer, err := p.ds.LatestVersion() lVer, err := p.ds.LatestVersion()
if err != datastore.ErrNotFound && p.errHandled(err) { if err != datastore.ErrNotFound && p.errHandled(err) {
@ -170,30 +198,33 @@ func (p *Project) fetch() {
} }
if p.version == lVer { if p.version == lVer {
// no new build // no new build clean up temp dir
p.errHandled(os.RemoveAll(tempDir))
vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
return return
} }
if p.errHandled(os.MkdirAll(p.verDir(), 0777)) { p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(p.version)))
//remove any existing data that matches version hash
if p.errHandled(os.RemoveAll(p.workingDir())) {
return return
} }
//fetch project //new version move tempdir to workingDir
fetchCmd := &exec.Cmd{ if p.errHandled(os.Rename(tempDir, p.workingDir())) {
Path: p.Fetch, // cleanup temp dir if rename failed
Dir: p.verDir(), p.errHandled(os.RemoveAll(tempDir))
}
fetchResult, err := fetchCmd.Output()
if p.errHandled(err) {
return return
} }
//log fetch results
if p.errHandled(p.ds.AddLog(p.stage, p.version, string(fetchResult))) { if p.errHandled(p.ds.AddLog(p.stage, p.version, string(fetchResult))) {
return return
} }
// fetched succesfully, onto the build stage // continue to build
p.build() p.build()
} }
@ -202,13 +233,9 @@ func (p *Project) fetch() {
// configured in the ReleaseFile section of the project file // configured in the ReleaseFile section of the project file
func (p *Project) build() { func (p *Project) build() {
p.stage = stageBuild p.stage = stageBuild
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
buildCmd := &exec.Cmd{ output, err := runCmd(p.Build, p.workingDir())
Path: p.Build,
Dir: p.verDir(),
}
output, err := buildCmd.Output()
if p.errHandled(err) { if p.errHandled(err) {
return return
@ -225,13 +252,9 @@ func (p *Project) build() {
// test runs the test scripts // test runs the test scripts
func (p *Project) test() { func (p *Project) test() {
p.stage = stageTest p.stage = stageTest
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
testCmd := &exec.Cmd{ output, err := runCmd(p.Test, p.workingDir())
Path: p.Test,
Dir: p.verDir(),
}
output, err := testCmd.Output()
if p.errHandled(err) { if p.errHandled(err) {
return return
@ -249,13 +272,9 @@ func (p *Project) test() {
// release runs the release scripts and builds the release file // release runs the release scripts and builds the release file
func (p *Project) release() { func (p *Project) release() {
p.stage = stageRelease p.stage = stageRelease
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
releaseCmd := &exec.Cmd{ output, err := runCmd(p.Release, p.workingDir())
Path: p.Release,
Dir: p.verDir(),
}
output, err := releaseCmd.Output()
if p.errHandled(err) { if p.errHandled(err) {
return return
@ -266,7 +285,7 @@ func (p *Project) release() {
} }
//get release file //get release file
f, err := os.Open(filepath.Join(p.verDir(), p.ReleaseFile)) f, err := os.Open(filepath.Join(p.workingDir(), p.ReleaseFile))
if p.errHandled(err) { if p.errHandled(err) {
return return
} }
@ -280,4 +299,12 @@ func (p *Project) release() {
return return
} }
//build successfull, remove working dir
p.errHandled(os.RemoveAll(p.workingDir()))
if p.errHandled(p.ds.Close()) {
return
}
vlog("Project: %s Version %s built, tested, and released successfully.\n", p.id(), p.version)
} }

View File

@ -64,7 +64,7 @@ func Open(filename string) (*Store, error) {
// Close closes the bolt datastore // Close closes the bolt datastore
func (ds *Store) Close() error { func (ds *Store) Close() error {
return ds.Close() return ds.bolt.Close()
} }
func (ds *Store) get(bucket string, key TimeKey, result interface{}) error { func (ds *Store) get(bucket string, key TimeKey, result interface{}) error {

View File

@ -34,7 +34,7 @@ func (ds *Store) AddLog(version, stage, entry string) error {
return ds.put(bucketLog, key, data) return ds.put(bucketLog, key, data)
} }
// LatestVersion returns the latest version for the current project // LatestVersion returns the latest version (successful or otherwise) for the current project
func (ds *Store) LatestVersion() (string, error) { func (ds *Store) LatestVersion() (string, error) {
version := "" version := ""

33
exec.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package main
import (
"fmt"
"os/exec"
"strings"
)
func runCmd(cmd, dir string) ([]byte, error) {
s := strings.Fields(cmd)
var args []string
if len(s) > 1 {
args = s[1:]
}
ec := exec.Command(s[0], args...)
ec.Dir = dir
vlog("Executing command: %s in dir %s\n", cmd, dir)
result, err := ec.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("%s:\n%s", err, result)
}
return result, nil
}

13
log.go Normal file
View File

@ -0,0 +1,13 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package main
import "log"
func vlog(format string, v ...interface{}) {
if verbose {
log.Printf(format, v...)
}
}

45
main.go
View File

@ -5,8 +5,10 @@
package main package main
import ( import (
"flag"
"log" "log"
"os" "os"
"os/signal"
"path/filepath" "path/filepath"
"git.townsourced.com/config" "git.townsourced.com/config"
@ -16,23 +18,46 @@ import (
var ( var (
projectDir = "./projects" // /etc/ironsmith/ projectDir = "./projects" // /etc/ironsmith/
dataDir = "./data" // /var/ironsmith/ dataDir = "./data" // /var/ironsmith/
address = "http://localhost:8026" address = ":8026"
certFile = "" certFile = ""
keyFile = "" keyFile = ""
) )
//flags
var (
verbose = false
)
func init() {
flag.BoolVar(&verbose, "v", false, "Verbose prints to stdOut every command and stage as it processes")
//Capture program shutdown, to make sure everything shuts down nicely
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
for sig := range c {
if sig == os.Interrupt {
projects.closeAll()
os.Exit(0)
}
}
}()
}
func main() { func main() {
flag.Parse()
settingPaths := config.StandardFileLocations("ironsmith/settings.json") settingPaths := config.StandardFileLocations("ironsmith/settings.json")
log.Println("IronSmith will use settings files in the following locations (in order of priority):") vlog("IronSmith will use settings files in the following locations (in order of priority):\n")
for i := range settingPaths { for i := range settingPaths {
log.Println("\t", settingPaths[i]) vlog("\t%s\n", settingPaths[i])
} }
cfg, err := config.LoadOrCreate(settingPaths...) cfg, err := config.LoadOrCreate(settingPaths...)
if err != nil { if err != nil {
log.Fatalf("Error loading or creating IronSmith settings file: %s", err) log.Fatalf("Error loading or creating IronSmith settings file: %s", err)
} }
log.Printf("IronSmith is currently using the file %s for settings.\n", cfg.FileName()) vlog("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
projectDir = cfg.String("projectDir", projectDir) projectDir = cfg.String("projectDir", projectDir)
dataDir = cfg.String("dataDir", dataDir) dataDir = cfg.String("dataDir", dataDir)
@ -40,6 +65,9 @@ func main() {
certFile = cfg.String("certFile", certFile) certFile = cfg.String("certFile", certFile)
keyFile = cfg.String("keyFile", keyFile) keyFile = cfg.String("keyFile", keyFile)
vlog("Project Definition Directory: %s\n", projectDir)
vlog("Project Data Directory: %s\n", dataDir)
//prep dirs //prep dirs
err = os.MkdirAll(filepath.Join(projectDir, enabledProjectDir), 0777) err = os.MkdirAll(filepath.Join(projectDir, enabledProjectDir), 0777)
if err != nil { if err != nil {
@ -61,5 +89,12 @@ func main() {
if err != nil { if err != nil {
log.Fatalf("Error loading projects: %s", err) log.Fatalf("Error loading projects: %s", err)
} }
//start server
//start web server
err = startServer()
if err != nil {
log.Fatalf("Error Starting web server: %s", err)
}
} }

View File

@ -59,6 +59,7 @@ type Project struct {
ds *datastore.Store ds *datastore.Store
stage string stage string
version string version string
hash string
} }
const projectTemplateFilename = "template.project.json" const projectTemplateFilename = "template.project.json"
@ -66,12 +67,12 @@ const projectTemplateFilename = "template.project.json"
var projectTemplate = &Project{ var projectTemplate = &Project{
Name: "Template Project", Name: "Template Project",
Fetch: "git clone root@git.townsourced.com:tshannon/ironsmith.git .", Fetch: "git clone root@git.townsourced.com:tshannon/ironsmith.git .",
Build: "sh ./ironsmith/build.sh", Build: "go build -o ironsmith",
Test: "sh ./ironsmith/test.sh", Test: "go test ./...",
Release: "sh ./ironsmith/release.sh", Release: "tar -czf release.tar.gz ironsmith",
Version: "git describe --tags --long", Version: "git describe --tags --long",
ReleaseFile: `json:"./ironsmith/release.tar.gz"`, ReleaseFile: "release.tar.gz",
PollInterval: "15m", PollInterval: "15m",
} }
@ -79,6 +80,7 @@ func prepTemplateProject() error {
filename := filepath.Join(projectDir, projectTemplateFilename) filename := filepath.Join(projectDir, projectTemplateFilename)
_, err := os.Stat(filename) _, err := os.Stat(filename)
if os.IsNotExist(err) { if os.IsNotExist(err) {
vlog("Creating template project file in %s", filename)
f, err := os.Create(filename) f, err := os.Create(filename)
defer func() { defer func() {
if cerr := f.Close(); cerr != nil && err == nil { if cerr := f.Close(); cerr != nil && err == nil {
@ -117,6 +119,7 @@ var projects = projectList{
} }
func (p *projectList) load() error { func (p *projectList) load() error {
vlog("Loading projects from the enabled definitions in %s\n", filepath.Join(projectDir, enabledProjectDir))
dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir)) dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
defer func() { defer func() {
if cerr := dir.Close(); cerr != nil && err == nil { if cerr := dir.Close(); cerr != nil && err == nil {
@ -153,6 +156,7 @@ func (p *projectList) exists(name string) bool {
} }
func (p *projectList) add(name string) { func (p *projectList) add(name string) {
vlog("Adding project %s to the project list.\n", name)
p.Lock() p.Lock()
defer p.Unlock() defer p.Unlock()
@ -181,11 +185,22 @@ func (p *projectList) removeMissing(names []string) {
} }
} }
if !found { if !found {
vlog("Removing project %s from the project list, because the project file was removed.\n",
i)
delete(p.data, i) delete(p.data, i)
} }
} }
} }
func (p *projectList) closeAll() {
p.RLock()
defer p.RUnlock()
for i := range p.data {
_ = p.data[i].ds.Close()
}
}
// startProjectLoader polls for new projects // startProjectLoader polls for new projects
func startProjectLoader() { func startProjectLoader() {
dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir)) dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
@ -196,13 +211,13 @@ func startProjectLoader() {
}() }()
if err != nil { if err != nil {
log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", dir, err) log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", projectDir, err)
return return
} }
files, err := dir.Readdir(0) files, err := dir.Readdir(0)
if err != nil { if err != nil {
log.Printf("Error in startProjectLoader reading the dir %s: %s\n", dir, err) log.Printf("Error in startProjectLoader reading the dir %s: %s\n", projectDir, err)
return return
} }

77
server.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package main
import "net/http"
var webRoot *http.ServeMux
func startServer() error {
var err error
routes()
server := &http.Server{
Handler: webRoot,
Addr: address,
}
if certFile == "" || keyFile == "" {
err = server.ListenAndServe()
} else {
server.Addr = address
err = server.ListenAndServeTLS(certFile, keyFile)
}
if err != nil {
return err
}
return nil
}
type methodHandler struct {
get http.HandlerFunc
post http.HandlerFunc
put http.HandlerFunc
delete http.HandlerFunc
}
func (m *methodHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if m.get == nil {
m.get = http.NotFound
}
if m.post == nil {
m.post = http.NotFound
}
if m.put == nil {
m.put = http.NotFound
}
if m.delete == nil {
m.delete = http.NotFound
}
switch r.Method {
case "GET":
m.get(w, r)
return
case "POST":
m.post(w, r)
return
case "PUT":
m.put(w, r)
return
case "DELETE":
m.delete(w, r)
return
default:
http.NotFound(w, r)
return
}
}
func routes() {
webRoot = http.NewServeMux()
}