// Copyright 2016 Tim Shannon. All rights reserved. // Use of this source code is governed by the MIT license // that can be found in the LICENSE file. package main import ( "crypto/sha1" "encoding/json" "fmt" "log" "os" "path/filepath" "strings" "sync" "time" "git.townsourced.com/townsourced/ironsmith/datastore" ) const ( enabledProjectDir = "enabled" deletedProjectDir = "deleted" ) //stages const ( stageLoad = "loading" stageFetch = "fetching" stageBuild = "building" stageTest = "testing" stageRelease = "releasing" stageReleased = "released" stageWait = "waiting" ) const projectFilePoll = 30 * time.Second // Project is an ironsmith project that contains how to fetch, build, test, and release a project /* The project lifecycle goes like this, each step calling the next if successful (Load Project file) -> (Fetch) -> (Build) -> (Test) -> (Release) - > (Sleep for polling period) -> (Reload Project File) -> (Fetch) -> etc... Changes the project file will be reloaded on every poll / trigger If a project file is deleted then the cycle will finish it's current poll and stop at the load phase */ type Project struct { Name string `json:"name"` // name of the project Environment []string `json:"environment"` // Environment for each of the scripts below, if empty will use the current processes environment Fetch string `json:"fetch"` //Script to fetch the latest project code into the current directory Build string `json:"build"` //Script to build the latest project code Test string `json:"test"` //Script to test the latest project code Release string `json:"release"` //Script to build the release of latest project code Version string `json:"version"` //Script to generate the version num of the current build, should be indempotent ReleaseFile string `json:"releaseFile"` PollInterval string `json:"pollInterval,omitempty"` // if not poll interval is specified, this project is trigger only TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call MaxVersions int `json:"maxVersions,omitempty"` // Max number of versions to keep in the project datastore filename string poll time.Duration ds *datastore.Store stage string status string version string hash string start time.Time // the last start time of the latest cycle sync.RWMutex processing sync.Mutex } func (p *Project) errHandled(err error) bool { if err == nil { return false } vlog("Error in project %s: %s\n", p.id(), err) if p.ds == nil { log.Printf("Error in project %s: %s\n", p.id(), err) return true } defer func() { //clean up version folder if it exists if p.version != "" { err = os.RemoveAll(p.workingDir()) if err != nil { log.Printf("Error deleting the version directory project %s version %s: %s\n", p.id(), p.version, err) } } }() lerr := p.ds.AddLog(p.version, p.stage, err.Error()) if lerr != nil { log.Printf("Error logging an error in project %s: Original error %s, Logging Error: %s", p.id(), err, lerr) } return true } func projectID(filename string) string { return strings.TrimSuffix(filename, filepath.Ext(filename)) } func (p *Project) id() string { if p.filename == "" { panic("invalid project filename") } return projectID(p.filename) } func (p *Project) dir() string { return filepath.Join(dataDir, p.id()) } func (p *Project) workingDir() string { if p.hash == "" { panic(fmt.Sprintf("Working dir called with no version hash set for project %s", p.id())) } //It's probably overkill to use a sha1 hash to identify the build folder, when putting a simple // timestamp on instead would work just fine, but I like having the working dir tied directly to the // version returned by project script return filepath.Join(p.dir(), p.hash) } // prepData makes sure the project's data folder and data store is created /* folder structure projectDataFolder// */ func (p *Project) open() error { p.Lock() defer p.Unlock() if p.ds != nil { return nil } err := os.MkdirAll(p.dir(), 0777) if err != nil { return err } ds, err := datastore.Open(filepath.Join(p.dir(), p.id()+".ironsmith")) if err != nil { return err } p.ds = ds return nil } func (p *Project) setVersion(version string) { p.Lock() defer p.Unlock() p.version = version if version == "" { p.hash = "" return } p.hash = fmt.Sprintf("%x", sha1.Sum([]byte(version))) } func (p *Project) setStage(stage string) { p.Lock() defer p.Unlock() if p.version != "" { vlog("Entering %s stage for Project: %s Version: %s\n", stage, p.id(), p.version) } else { vlog("Entering %s stage for Project: %s\n", stage, p.id()) } p.stage = stage } type webProject struct { ID string `json:"id"` Name string `json:"name"` ReleaseVersion string `json:"releaseVersion"` //last successfully released version Stage string `json:"stage"` // current stage LastLog *datastore.Log `json:"lastLog"` } func (p *Project) webData() (*webProject, error) { p.RLock() defer p.RUnlock() last, err := p.ds.LastVersion("") if err != nil { return nil, err } release, err := p.ds.LastVersion(stageRelease) if err != nil { return nil, err } d := &webProject{ Name: p.Name, ID: p.id(), ReleaseVersion: release.Version, Stage: p.stage, LastLog: last, } return d, nil } func (p *Project) versions() ([]*datastore.Log, error) { p.RLock() defer p.RUnlock() return p.ds.Versions() } func (p *Project) versionLog(version string) ([]*datastore.Log, error) { p.RLock() defer p.RUnlock() return p.ds.VersionLog(version) } func (p *Project) stageLog(version, stage string) (*datastore.Log, error) { p.RLock() defer p.RUnlock() return p.ds.StageLog(version, stage) } func (p *Project) releases() ([]*datastore.Release, error) { p.RLock() defer p.RUnlock() return p.ds.Releases() } func (p *Project) lastRelease() (*datastore.Release, error) { p.RLock() defer p.RUnlock() return p.ds.LastRelease() } func (p *Project) releaseData(version string) (*datastore.Release, error) { p.RLock() defer p.RUnlock() return p.ds.Release(version) } func (p *Project) releaseFile(fileKey datastore.TimeKey) ([]byte, error) { p.RLock() defer p.RUnlock() return p.ds.ReleaseFile(fileKey) } // releaseFile func (p *Project) setData(new *Project) { p.Lock() defer p.Unlock() p.Name = new.Name p.Environment = new.Environment p.Fetch = new.Fetch p.Build = new.Build p.Test = new.Test p.Release = new.Release p.Version = new.Version p.ReleaseFile = new.ReleaseFile p.PollInterval = new.PollInterval p.TriggerSecret = new.TriggerSecret p.MaxVersions = new.MaxVersions if p.PollInterval != "" { var err error p.poll, err = time.ParseDuration(p.PollInterval) if p.errHandled(err) { p.poll = 0 } } } func (p *Project) close() error { p.Lock() defer p.Unlock() if p.ds == nil { return nil } err := p.ds.Close() if err != nil { return err } p.ds = nil return nil } const projectTemplateFilename = "template.project.json" var projectTemplate = &Project{ Name: "Template Project", Fetch: "git clone root@git.townsourced.com:tshannon/ironsmith.git .", Build: "go build -a -v -o ironsmith", Test: "go test ./...", Release: "tar -czf release.tar.gz ironsmith", Version: "git describe --tags --long", ReleaseFile: "release.tar.gz", PollInterval: "15m", MaxVersions: 100, } func prepTemplateProject() error { filename := filepath.Join(projectDir, projectTemplateFilename) _, err := os.Stat(filename) if os.IsNotExist(err) { vlog("Creating template project file in %s", filename) f, err := os.Create(filename) defer func() { if cerr := f.Close(); cerr != nil && err == nil { err = cerr } }() if err != nil { return err } data, err := json.MarshalIndent(projectTemplate, "", " ") if err != nil { return err } _, err = f.Write(data) if err != nil { return err } } else if err != nil { return err } return nil } type projectList struct { sync.RWMutex data map[string]*Project } var projects = projectList{ data: make(map[string]*Project), } func (p *projectList) load() error { vlog("Loading projects from the enabled definitions in %s\n", filepath.Join(projectDir, enabledProjectDir)) dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir)) defer func() { if cerr := dir.Close(); cerr != nil && err == nil { err = cerr } }() if err != nil { return err } files, err := dir.Readdir(0) if err != nil { return err } for i := range files { if !files[i].IsDir() && filepath.Ext(files[i].Name()) == ".json" { p.add(files[i].Name()) } } time.AfterFunc(projectFilePoll, startProjectLoader) return nil } func (p *projectList) get(name string) (*Project, bool) { p.RLock() defer p.RUnlock() prj, ok := p.data[name] return prj, ok } func (p *projectList) add(name string) { vlog("Adding project %s to the project list.\n", name) p.Lock() defer p.Unlock() prj := &Project{ filename: name, Name: name, stage: stageLoad, } p.data[projectID(name)] = prj go func() { err := prj.open() if err != nil { log.Printf("Error opening datastore for Project: %s Error: %s\n", prj.id(), err) return } prj.load(false) }() } // removeMissing removes projects that are missing from the passed in list of names func (p *projectList) removeMissing(names []string) { p.Lock() defer p.Unlock() for i := range p.data { found := false for k := range names { if projectID(names[k]) == i { found = true } } if !found { vlog("Removing project %s from the project list, because the project file was removed.\n", i) delete(p.data, i) } } } func (p *projectList) stopAll() { p.RLock() defer p.RUnlock() for i := range p.data { err := p.data[i].close() if err != nil { log.Printf("Error closing project datastore for Project: %s Error: %s\n", p.data[i].id(), err) } } } func (p *projectList) webList() ([]*webProject, error) { p.RLock() defer p.RUnlock() list := make([]*webProject, 0, len(p.data)) for i := range p.data { prj, err := p.data[i].webData() if err != nil { return nil, err } list = append(list, prj) } return list, nil } // startProjectLoader polls for new projects func startProjectLoader() { dir, err := os.Open(filepath.Join(projectDir, enabledProjectDir)) defer func() { if cerr := dir.Close(); cerr != nil && err == nil { err = cerr } }() if err != nil { log.Printf("Error in startProjectLoader opening the filepath %s: %s\n", projectDir, err) return } files, err := dir.Readdir(0) if err != nil { log.Printf("Error in startProjectLoader reading the dir %s: %s\n", projectDir, err) return } names := make([]string, len(files)) for i := range files { if !files[i].IsDir() && filepath.Ext(files[i].Name()) == ".json" { names[i] = files[i].Name() if _, ok := projects.get(projectID(files[i].Name())); !ok { projects.add(files[i].Name()) } } } //check for removed projects projects.removeMissing(names) time.AfterFunc(projectFilePoll, startProjectLoader) }