townsourced/ironsmith
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added logging and proper execution of commands

Browse Source 
Fixed several issues, and ran a basic test on ironsmith itself.

Need to prevent subsequent builds on same versions
This commit is contained in:
Tim Shannon 2016-04-05 21:59:52 +00:00
parent 38d20d46fe
commit 65c489c920
8 changed files with 259 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
cycle.go
View File

@ -5,13 +5,15 @@
package main
import (
"crypto/sha1"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
@ -23,6 +25,8 @@ func (p *Project) errHandled(err error) bool {
return false
}
vlog("Error in project %s: %s\n", p.id(), err)
if p.ds == nil {
log.Printf("Error in project %s: %s\n", p.id(), err)
return true
@ -37,9 +41,11 @@ func (p *Project) errHandled(err error) bool {
//clean up version folder if it exists
if p.version != "" {
err = os.RemoveAll(p.verDir())
log.Printf("Error deleting the version directory project %s version %s: %s\n",
p.id(), p.version, err)
err = os.RemoveAll(p.workingDir())
if err != nil {
log.Printf("Error deleting the version directory project %s version %s: %s\n",
p.id(), p.version, err)
}
}
}()
@ -64,8 +70,16 @@ func (p *Project) dir() string {
return filepath.Join(dataDir, p.id())
}
func (p *Project) verDir() string {
return filepath.Join(p.dir(), p.version)
func (p *Project) workingDir() string {
if p.hash == "" {
panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id()))
}
//It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple
// timestamp on instead would work just fine, but I like having the working dir tied directly to the
// version returned by project script
return filepath.Join(p.dir(), p.hash)
}
// prepData makes sure the project's data folder and data store is created
@ -100,6 +114,9 @@ Project life cycle:
// call's fetch and triggers the next poll if one exists
func (p *Project) load() {
p.version = ""
p.hash = ""
vlog("Entering %s stage for Project: %s\n", stageLoad, p.id())
if p.filename == "" {
p.errHandled(errors.New("Invalid project file name"))
@ -109,8 +126,9 @@ func (p *Project) load() {
if !projects.exists(p.filename) {
// project has been deleted
// don't continue polling
// move project data to deleted folder
p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir, p.id())))
// move project data to deleted folder with a timestamp
p.errHandled(os.Rename(p.dir(), filepath.Join(dataDir, deletedProjectDir,
strconv.FormatInt(time.Now().Unix(), 10), p.id())))
return
}
@ -146,23 +164,33 @@ func (p *Project) load() {
}
}
// fetch first runs the version script and checks the returned version against the latest version in the
// project database. If the version hasn't changed, then it breaks out of the cycle early doing nothing
// if the version has changed, then it runs the fetch script
// fetch first runs the fetch script into a temporary directory
// then it runs the version script in the temp directory to see if there is a newer version of the
// fetched code, if there is then the temp dir is renamed to the version name
func (p *Project) fetch() {
p.stage = stageFetch
verCmd := &exec.Cmd{
Path: p.Version,
Dir: p.dir(),
vlog("Entering %s stage for Project: %s\n", p.stage, p.id())
tempDir := filepath.Join(p.dir(), strconv.FormatInt(time.Now().Unix(), 10))
if p.errHandled(os.MkdirAll(tempDir, 0777)) {
return
}
version, err := verCmd.Output()
//fetch project
fetchResult, err := runCmd(p.Fetch, tempDir)
if p.errHandled(err) {
return
}
// fetched succesfully, determine version
version, err := runCmd(p.Version, tempDir)
if p.errHandled(err) {
return
}
p.version = string(version)
p.version = strings.TrimSpace(string(version))
lVer, err := p.ds.LatestVersion()
if err != datastore.ErrNotFound && p.errHandled(err) {
@ -170,30 +198,33 @@ func (p *Project) fetch() {
}
if p.version == lVer {
// no new build
// no new build clean up temp dir
p.errHandled(os.RemoveAll(tempDir))
vlog("No new version found for Project: %s Version: %s.\n", p.id(), p.version)
return
}
if p.errHandled(os.MkdirAll(p.verDir(), 0777)) {
p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(p.version)))
//remove any existing data that matches version hash
if p.errHandled(os.RemoveAll(p.workingDir())) {
return
}
//fetch project
fetchCmd := &exec.Cmd{
Path: p.Fetch,
Dir: p.verDir(),
}
fetchResult, err := fetchCmd.Output()
if p.errHandled(err) {
//new version move tempdir to workingDir
if p.errHandled(os.Rename(tempDir, p.workingDir())) {
// cleanup temp dir if rename failed
p.errHandled(os.RemoveAll(tempDir))
return
}
//log fetch results
if p.errHandled(p.ds.AddLog(p.stage, p.version, string(fetchResult))) {
return
}
// fetched succesfully, onto the build stage
// continue to build
p.build()
}
@ -202,13 +233,9 @@ func (p *Project) fetch() {
// configured in the ReleaseFile section of the project file
func (p *Project) build() {
p.stage = stageBuild
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
buildCmd := &exec.Cmd{
Path: p.Build,
Dir: p.verDir(),
}
output, err := buildCmd.Output()
output, err := runCmd(p.Build, p.workingDir())
if p.errHandled(err) {
return
@ -225,13 +252,9 @@ func (p *Project) build() {
// test runs the test scripts
func (p *Project) test() {
p.stage = stageTest
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
testCmd := &exec.Cmd{
Path: p.Test,
Dir: p.verDir(),
}
output, err := testCmd.Output()
output, err := runCmd(p.Test, p.workingDir())
if p.errHandled(err) {
return
@ -249,13 +272,9 @@ func (p *Project) test() {
// release runs the release scripts and builds the release file
func (p *Project) release() {
p.stage = stageRelease
vlog("Entering %s stage for Project: %s Version: %s\n", p.stage, p.id(), p.version)
releaseCmd := &exec.Cmd{
Path: p.Release,
Dir: p.verDir(),
}
output, err := releaseCmd.Output()
output, err := runCmd(p.Release, p.workingDir())
if p.errHandled(err) {
return
@ -266,7 +285,7 @@ func (p *Project) release() {
}
//get release file
f, err := os.Open(filepath.Join(p.verDir(), p.ReleaseFile))
f, err := os.Open(filepath.Join(p.workingDir(), p.ReleaseFile))
if p.errHandled(err) {
return
}
@ -280,4 +299,12 @@ func (p *Project) release() {
return
}
//build successfull, remove working dir
p.errHandled(os.RemoveAll(p.workingDir()))
if p.errHandled(p.ds.Close()) {
return
}
vlog("Project: %s Version %s built, tested, and released successfully.\n", p.id(), p.version)
}

2
datastore/ds.go
View File

@ -64,7 +64,7 @@ func Open(filename string) (*Store, error) {
// Close closes the bolt datastore
func (ds *Store) Close() error {
return ds.Close()
return ds.bolt.Close()
}
func (ds *Store) get(bucket string, key TimeKey, result interface{}) error {

2
datastore/log.go
View File

@ -34,7 +34,7 @@ func (ds *Store) AddLog(version, stage, entry string) error {
return ds.put(bucketLog, key, data)
}
// LatestVersion returns the latest version for the current project
// LatestVersion returns the latest version (successful or otherwise) for the current project
func (ds *Store) LatestVersion() (string, error) {
version := ""

33
exec.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package main
import (
"fmt"
"os/exec"
"strings"
)
func runCmd(cmd, dir string) ([]byte, error) {
s := strings.Fields(cmd)
var args []string
if len(s) > 1 {
args = s[1:]
}
ec := exec.Command(s[0], args...)
ec.Dir = dir
vlog("Executing command: %s in dir %s\n", cmd, dir)
result, err := ec.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("%s:\n%s", err, result)
}
return result, nil
}

13
log.go Normal file
View File

@ -0,0 +1,13 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package main
import "log"
func vlog(format string, v ...interface{}) {
if verbose {
log.Printf(format, v...)
}
}

45
main.go
View File

@ -5,8 +5,10 @@
package main
import (
"flag"
"log"
"os"
"os/signal"
"path/filepath"
"git.townsourced.com/config"
@ -16,23 +18,46 @@ import (
var (
projectDir = "./projects" // /etc/ironsmith/
dataDir = "./data" // /var/ironsmith/
address = "http://localhost:8026"
address = ":8026"
certFile = ""
keyFile = ""
)
//flags
var (
verbose = false
)
func init() {
flag.BoolVar(&verbose, "v", false, "Verbose prints to stdOut every command and stage as it processes")
//Capture program shutdown, to make sure everything shuts down nicely
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
for sig := range c {
if sig == os.Interrupt {
projects.closeAll()
os.Exit(0)
}
}
}()
}
func main() {
flag.Parse()
settingPaths := config.StandardFileLocations("ironsmith/settings.json")
log.Println("IronSmith will use settings files in the following locations (in order of priority):")
vlog("IronSmith will use settings files in the following locations (in order of priority):\n")
for i := range settingPaths {
log.Println("\t", settingPaths[i])
vlog("\t%s\n", settingPaths[i])
}
cfg, err := config.LoadOrCreate(settingPaths...)
if err != nil {
log.Fatalf("Error loading or creating IronSmith settings file: %s", err)
}
log.Printf("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
vlog("IronSmith is currently using the file %s for settings.\n", cfg.FileName())
projectDir = cfg.String("projectDir", projectDir)
dataDir = cfg.String("dataDir", dataDir)
@ -40,6 +65,9 @@ func main() {
certFile = cfg.String("certFile", certFile)
keyFile = cfg.String("keyFile", keyFile)
vlog("Project Definition Directory: %s\n", projectDir)
vlog("Project Data Directory: %s\n", dataDir)
//prep dirs
err = os.MkdirAll(filepath.Join(projectDir, enabledProjectDir), 0777)
if err != nil {
@ -61,5 +89,12 @@ func main() {
if err != nil {
log.Fatalf("Error loading projects: %s", err)
}
//start server
//start web server
err = startServer()
if err != nil {
log.Fatalf("Error Starting web server: %s", err)
}
}

27
project.go
View File

@ -59,6 +59,7 @@ type Project struct {
ds *datastore.Store
stage string
version string
hash string
}
const projectTemplateFilename = "template.project.json"
@ -66,12 +67,12 @@ const projectTemplateFilename = "template.project.json"
var projectTemplate = &Project{
Name: "Template Project",
Fetch: "git clone root@git.townsourced.com:tshannon/ironsmith.git .",
Build: "sh ./ironsmith/build.sh",
Test: "sh ./ironsmith/test.sh",
Release: "sh ./ironsmith/release.sh",
Build: "go build -o ironsmith",
Test: "go test ./...",
Release: "tar -czf release.tar.gz ironsmith",
Version: "git describe --tags --long",
ReleaseFile: `json:"./ironsmith/release.tar.gz"`,
ReleaseFile: "release.tar.gz",
PollInterval: "15m",
}
@ -79,6 +80,7 @@ func prepTemplateProject() error {
filename := filepath.Join(projectDir, projectTemplateFilename)
_, err := os.Stat(filename)
if os.IsNotExist(err) {
vlog("Creating template project file in %s", filename)
f, err := os.Create(filename)
defer func() {
if cerr := f.Close(); cerr != nil && err == nil {
@ -117,6 +119,7 @@ var projects = projectList{
}
func (p *projectList) load() error {
vlog("Loading projects from the enabled definitions in %s\n", filepath.Join(projectDir, enabledProjectDir))
dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
defer func() {
if cerr := dir.Close(); cerr != nil && err == nil {
@ -153,6 +156,7 @@ func (p *projectList) exists(name string) bool {
}
func (p *projectList) add(name string) {
vlog("Adding project %s to the project list.\n", name)
p.Lock()
defer p.Unlock()
@ -181,11 +185,22 @@ func (p *projectList) removeMissing(names []string) {
}
}
if !found {
vlog("Removing project %s from the project list, because the project file was removed.\n",
i)
delete(p.data, i)
}
}
}
func (p *projectList) closeAll() {
p.RLock()
defer p.RUnlock()
for i := range p.data {
_ = p.data[i].ds.Close()
}
}
// startProjectLoader polls for new projects
func startProjectLoader() {
dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir))
@ -196,13 +211,13 @@ func startProjectLoader() {
}()
if err != nil {
log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", dir, err)
log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", projectDir, err)
return
}
files, err := dir.Readdir(0)
if err != nil {
log.Printf("Error in startProjectLoader reading the dir %s: %s\n", dir, err)
log.Printf("Error in startProjectLoader reading the dir %s: %s\n", projectDir, err)
return
}

77
server.go Normal file
View File

@ -0,0 +1,77 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package main
import "net/http"
var webRoot *http.ServeMux
func startServer() error {
var err error
routes()
server := &http.Server{
Handler: webRoot,
Addr: address,
}
if certFile == "" || keyFile == "" {
err = server.ListenAndServe()
} else {
server.Addr = address
err = server.ListenAndServeTLS(certFile, keyFile)
}
if err != nil {
return err
}
return nil
}
type methodHandler struct {
get http.HandlerFunc
post http.HandlerFunc
put http.HandlerFunc
delete http.HandlerFunc
}
func (m *methodHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if m.get == nil {
m.get = http.NotFound
}
if m.post == nil {
m.post = http.NotFound
}
if m.put == nil {
m.put = http.NotFound
}
if m.delete == nil {
m.delete = http.NotFound
}
switch r.Method {
case "GET":
m.get(w, r)
return
case "POST":
m.post(w, r)
return
case "PUT":
m.put(w, r)
return
case "DELETE":
m.delete(w, r)
return
default:
http.NotFound(w, r)
return
}
}
func routes() {
webRoot = http.NewServeMux()
}