From 35ec6323341dfb5f3b4bbf6822158ad06794063a Mon Sep 17 00:00:00 2001 From: Tim Shannon Date: Wed, 4 May 2016 15:39:56 -0500 Subject: [PATCH] Added max version limit for projects --- cycle.go | 1 + datastore/ds.go | 83 +++++++++++++++++++++++++++++++++++++++---- datastore/releases.go | 32 ++++++++++++----- exec.go | 6 ++-- project.go | 5 ++- 5 files changed, 109 insertions(+), 18 deletions(-) diff --git a/cycle.go b/cycle.go index 23e4755..10731eb 100644 --- a/cycle.go +++ b/cycle.go @@ -69,6 +69,7 @@ func (p *Project) load() { p.setStage(stageWait) //full cycle completed + p.errHandled(p.ds.TrimVersions(p.MaxVersions)) if p.poll > 0 { //start polling diff --git a/datastore/ds.go b/datastore/ds.go index d0a0a8c..0ae6abf 100644 --- a/datastore/ds.go +++ b/datastore/ds.go @@ -67,6 +67,83 @@ func (ds *Store) Close() error { return ds.bolt.Close() } +// TrimVersions Removes versions from the datastore file until it reaches the maxVersions count +func (ds *Store) TrimVersions(maxVersions int) error { + if maxVersions <= 0 { + // no max set + return nil + } + + versions, err := ds.Versions() + if err != nil { + return err + } + + if len(versions) <= maxVersions { + return nil + } + + remove := versions[maxVersions:] + + for i := range remove { + err = ds.deleteVersion(remove[i].Version) + if err != nil { + return err + } + } + + return nil +} + +// removes the earliest instance of a specific version +func (ds *Store) deleteVersion(version string) error { + return ds.bolt.Update(func(tx *bolt.Tx) error { + // remove all logs for this version + c := tx.Bucket([]byte(bucketLog)).Cursor() + + for k, v := c.First(); k != nil; k, v = c.Next() { + lg := &Log{} + + err := json.Unmarshal(v, lg) + if err != nil { + return err + } + + if lg.Version != version { + break + } + + err = c.Delete() + if err != nil { + return err + } + } + + // remove all releases for this version + release, err := ds.Release(version) + if err == ErrNotFound { + return nil + } + if err != nil { + return err + } + + err = tx.Bucket([]byte(bucketReleases)).Delete(release.FileKey.Bytes()) + if err != nil { + return err + } + + // remove release file for this version + err = tx.Bucket([]byte(bucketFiles)).Delete(release.FileKey.Bytes()) + if err != nil { + return err + } + + return nil + }) + +} + func (ds *Store) get(bucket string, key []byte, result interface{}) error { return ds.bolt.View(func(tx *bolt.Tx) error { dsValue := tx.Bucket([]byte(bucket)).Get(key) @@ -93,9 +170,3 @@ func (ds *Store) put(bucket string, key []byte, value interface{}) error { return tx.Bucket([]byte(bucket)).Put(key, dsValue) }) } - -func (ds *Store) delete(bucket string, key []byte) error { - return ds.bolt.Update(func(tx *bolt.Tx) error { - return tx.Bucket([]byte(bucket)).Delete(key) - }) -} diff --git a/datastore/releases.go b/datastore/releases.go index c6b7e2e..f9a9259 100644 --- a/datastore/releases.go +++ b/datastore/releases.go @@ -28,13 +28,13 @@ const ( // AddRelease adds a new Release func (ds *Store) AddRelease(version, fileName string, fileData []byte) error { - fileKey := NewTimeKey() + key := NewTimeKey() r := &Release{ - When: fileKey.Time(), + When: key.Time(), Version: version, FileName: fileName, - FileKey: fileKey, + FileKey: key, } dsValue, err := json.Marshal(r) @@ -43,12 +43,12 @@ func (ds *Store) AddRelease(version, fileName string, fileData []byte) error { } return ds.bolt.Update(func(tx *bolt.Tx) error { - err = tx.Bucket([]byte(bucketReleases)).Put([]byte(version), dsValue) + err = tx.Bucket([]byte(bucketReleases)).Put(key.Bytes(), dsValue) if err != nil { return err } - return tx.Bucket([]byte(bucketFiles)).Put(fileKey.Bytes(), fileData) + return tx.Bucket([]byte(bucketFiles)).Put(key.Bytes(), fileData) }) } @@ -80,10 +80,27 @@ func (ds *Store) ReleaseFile(fileKey TimeKey) ([]byte, error) { // Release gets the release record for a specific version func (ds *Store) Release(version string) (*Release, error) { r := &Release{} - err := ds.get(bucketReleases, []byte(version), r) + err := ds.bolt.View(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte(bucketReleases)).Cursor() + + for k, v := c.Last(); k != nil; k, v = c.Prev() { + err := json.Unmarshal(v, r) + if err != nil { + return err + } + + if r.Version == version { + return nil + } + } + + return ErrNotFound + }) + if err != nil { return nil, err } + return r, nil } @@ -119,9 +136,8 @@ func (ds *Store) LastRelease() (*Release, error) { r := &Release{} err := ds.bolt.View(func(tx *bolt.Tx) error { - c := tx.Bucket([]byte(bucketReleases)).Cursor() + _, v := tx.Bucket([]byte(bucketReleases)).Cursor().Last() - _, v := c.First() // this is confusing if v == nil { return ErrNotFound } diff --git a/exec.go b/exec.go index 584cc81..b61ce3a 100644 --- a/exec.go +++ b/exec.go @@ -59,14 +59,14 @@ func lookPath(file string, env []string) (string, error) { if err == nil { return file, nil } - return "", &exec.Error{file, err} + return "", &exec.Error{Name: file, Err: err} } for i := range env { if strings.HasPrefix(env[i], "PATH=") { pathenv := env[i][5:] if pathenv == "" { - return "", &exec.Error{file, exec.ErrNotFound} + return "", &exec.Error{Name: file, Err: exec.ErrNotFound} } for _, dir := range strings.Split(pathenv, ":") { if dir == "" { @@ -78,7 +78,7 @@ func lookPath(file string, env []string) (string, error) { return path, nil } } - return "", &exec.Error{file, exec.ErrNotFound} + return "", &exec.Error{Name: file, Err: exec.ErrNotFound} } } diff --git a/project.go b/project.go index 31fd480..80e04e3 100644 --- a/project.go +++ b/project.go @@ -60,7 +60,7 @@ type Project struct { ReleaseFile string `json:"releaseFile"` PollInterval string `json:"pollInterval,omitempty"` // if not poll interval is specified, this project is trigger only TriggerSecret string `json:"triggerSecret,omitempty"` //secret to be included with a trigger call - MaxSizeInMB string `json:"maxSizeInMB,omitempty"` // Max size of the project datastore in MB + MaxVersions int `json:"maxVersions,omitempty"` // Max number of versions to keep in the project datastore filename string poll time.Duration @@ -288,6 +288,7 @@ func (p *Project) setData(new *Project) { p.ReleaseFile = new.ReleaseFile p.PollInterval = new.PollInterval p.TriggerSecret = new.TriggerSecret + p.MaxVersions = new.MaxVersions if p.PollInterval != "" { var err error @@ -325,6 +326,8 @@ var projectTemplate = &Project{ ReleaseFile: "release.tar.gz", PollInterval: "15m", + + MaxVersions: 100, } func prepTemplateProject() error {