From a5660ba38f2bfa67e2c22fc5fc5ada0eadbaa4ec Mon Sep 17 00:00:00 2001 From: Tim Shannon Date: Thu, 31 Mar 2016 21:59:48 +0000 Subject: [PATCH] Added datastore handling Started on basic data layer stuff --- datastore/ds.go | 108 ++++++++++++++++++++++++++++++++++++++++++ datastore/ds_test.go | 56 ++++++++++++++++++++++ datastore/key.go | 85 +++++++++++++++++++++++++++++++++ datastore/log.go | 30 ++++++++++++ datastore/releases.go | 42 ++++++++++++++++ project.go | 5 +- 6 files changed, 325 insertions(+), 1 deletion(-) create mode 100644 datastore/ds_test.go create mode 100644 datastore/key.go create mode 100644 datastore/log.go create mode 100644 datastore/releases.go diff --git a/datastore/ds.go b/datastore/ds.go index f597128..bcae7cd 100644 --- a/datastore/ds.go +++ b/datastore/ds.go @@ -4,3 +4,111 @@ // Package datastore manages the bolt data files and the reading and writing data to them package datastore + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "time" + + "github.com/boltdb/bolt" +) + +// ErrNotFound is the error returned when a value cannot be found in the store for the given key +var ErrNotFound = errors.New("Value not found") + +// Store is a datastore for getting and setting data for a given ironsmith project +// run on top of a Bolt DB file +type Store struct { + bolt *bolt.DB +} + +// Open opens an existing datastore file, or creates a new one +// caller is responsible for closing the datastore +func Open(filename string) (*Store, error) { + db, err := bolt.Open(filename, 0666, &bolt.Options{Timeout: 1 * time.Minute}) + + if err != nil { + return nil, err + } + + store := &Store{ + bolt: db, + } + + err = store.bolt.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(bucketLog)) + if err != nil { + return err + } + _, err = tx.CreateBucketIfNotExists([]byte(bucketReleases)) + if err != nil { + return err + } + + _, err = tx.CreateBucketIfNotExists([]byte(bucketFiles)) + if err != nil { + return err + } + + return nil + }) + + if err != nil { + return nil, err + } + + return store, nil +} + +// Close closes the bolt datastore +func (ds *Store) Close() error { + if ds != nil { + return ds.Close() + } + + return nil +} + +func (ds *Store) get(bucket string, key TimeKey, result interface{}) error { + return ds.bolt.View(func(tx *bolt.Tx) error { + dsValue := tx.Bucket([]byte(bucket)).Get(key.Bytes()) + + if dsValue == nil { + return ErrNotFound + } + + if value, ok := result.([]byte); ok { + buff := bytes.NewBuffer(value) + _, err := io.Copy(buff, bytes.NewReader(dsValue)) + if err != nil { + return err + } + return nil + } + + return json.Unmarshal(dsValue, result) + }) +} + +func (ds *Store) put(bucket string, key TimeKey, value interface{}) error { + var err error + dsValue, ok := value.([]byte) + if !ok { + dsValue, err = json.Marshal(value) + if err != nil { + return err + } + } + + return ds.bolt.Update(func(tx *bolt.Tx) error { + return tx.Bucket([]byte(bucket)).Put(key.Bytes(), dsValue) + }) +} + +func (ds *Store) delete(bucket string, key TimeKey) error { + return ds.bolt.Update(func(tx *bolt.Tx) error { + return tx.Bucket([]byte(bucket)).Delete(key.Bytes()) + }) +} diff --git a/datastore/ds_test.go b/datastore/ds_test.go new file mode 100644 index 0000000..5dd85a6 --- /dev/null +++ b/datastore/ds_test.go @@ -0,0 +1,56 @@ +// Copyright 2016 Tim Shannon. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package datastore + +import ( + "sort" + "testing" + "time" +) + +func TestNewTimeKey(t *testing.T) { + tk := NewTimeKey() + + if len(tk) != (128 / 8) { + t.Errorf("Invalid Time Key Length want %d got %d", 128/8, len(tk)) + } +} + +func TestTimeKeyTime(t *testing.T) { + + now := time.Now() + + tk := NewTimeKey() + + tkTime := tk.Time() + + if !tkTime.After(now) && !tkTime.Equal(now) { + t.Errorf("TimeKey's time is not after or equal a previous generated timestamp. want: %s, got: %s", now, tkTime) + } + + cTime := tk.Time() + + if !tkTime.Equal(cTime) { + t.Errorf("TimeKey's time is not consistently parsed from the timekey. want: %s, got: %s", tkTime, cTime) + } +} + +type ByTime []TimeKey + +func (a ByTime) Len() int { return len(a) } +func (a ByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByTime) Less(i, j int) bool { return a[i].Time().Before(a[j].Time()) } + +func TestTimeKeyOrder(t *testing.T) { + keys := make([]TimeKey, 1000) + for i := range keys { + keys[i] = NewTimeKey() + } + + if !sort.IsSorted(ByTime(keys)) { + t.Errorf("TimeKey's are not properly sorted by time") + } + +} diff --git a/datastore/key.go b/datastore/key.go new file mode 100644 index 0000000..4b7517c --- /dev/null +++ b/datastore/key.go @@ -0,0 +1,85 @@ +// Copyright 2016 Tim Shannon. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package datastore + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "time" +) + +// TimeKey is a unique time ordered key for use in the datastore +// A TimeKey is 32 bits random data + 96 bit UNIX timestamp (64bits seconds + 32 bit nanoseconds) +type TimeKey [16]byte + +// NewTimeKey returns a newly generated TimeKey based on the current time +func NewTimeKey() TimeKey { + rBits := make([]byte, 32/8) + _, err := io.ReadFull(rand.Reader, rBits) + if err != nil { + panic(fmt.Sprintf("Error generating random values for New TimeKey: %v", err)) + } + + t := time.Now() + sec := t.Unix() + nsec := t.Nanosecond() + + return TimeKey{ + rBits[0], //random + rBits[1], + rBits[2], + rBits[3], + byte(sec >> 56), // seconds + byte(sec >> 48), + byte(sec >> 40), + byte(sec >> 32), + byte(sec >> 24), + byte(sec >> 16), + byte(sec >> 8), + byte(sec), + byte(nsec >> 24), // nanoseconds + byte(nsec >> 16), + byte(nsec >> 8), + byte(nsec), + } +} + +// Time returns the time portion of a timekey +func (k TimeKey) Time() time.Time { + buf := k[4:] + + sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 | + int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56 + + buf = buf[8:] + nsec := int32(buf[3]) | int32(buf[2])<<8 | int32(buf[1])<<16 | int32(buf[0])<<24 + + return time.Unix(sec, int64(nsec)) +} + +// UUID returns the string representation of a TimeKey in Hex format separated by dashes +// similar to a UUID xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +func (k TimeKey) UUID() string { + buf := make([]byte, 36) + + hex.Encode(buf[0:8], k[0:4]) + buf[8] = '-' + hex.Encode(buf[9:13], k[4:6]) + buf[13] = '-' + hex.Encode(buf[14:18], k[6:8]) + buf[18] = '-' + hex.Encode(buf[19:23], k[8:10]) + buf[23] = '-' + hex.Encode(buf[24:], k[10:]) + + return string(buf) +} + +// Bytes returns the a slice of the underlying bytes of a TimeKey +func (k TimeKey) Bytes() []byte { + return []byte(k[:]) +} diff --git a/datastore/log.go b/datastore/log.go new file mode 100644 index 0000000..7fdf5c0 --- /dev/null +++ b/datastore/log.go @@ -0,0 +1,30 @@ +// Copyright 2016 Tim Shannon. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package datastore + +import "time" + +type log struct { + When time.Time `json:"when"` + Version string `json:"version"` + Stage string `json:"stage"` + Log string `json:"log"` +} + +const bucketLog = "log" + +// AddLog adds a new log entry +func (ds *Store) AddLog(version, stage, entry string) error { + key := NewTimeKey() + + data := &log{ + When: key.Time(), + Version: version, + Stage: stage, + Log: entry, + } + + return ds.put(bucketLog, key, data) +} diff --git a/datastore/releases.go b/datastore/releases.go new file mode 100644 index 0000000..bee1c64 --- /dev/null +++ b/datastore/releases.go @@ -0,0 +1,42 @@ +// Copyright 2016 Tim Shannon. All rights reserved. +// Use of this source code is governed by the MIT license +// that can be found in the LICENSE file. + +package datastore + +import "time" + +type release struct { + When time.Time `json:"when"` + Version string `json:"version"` + FileKey TimeKey `json:"file"` +} + +const ( + bucketReleases = "releases" + bucketFiles = "files" +) + +// AddRelease adds a new Release +func (ds *Store) AddRelease(version string, fileData []byte) error { + key := NewTimeKey() + fileKey := NewTimeKey() + + r := &release{ + When: key.Time(), + Version: version, + FileKey: fileKey, + } + + err := ds.put(bucketReleases, key, r) + if err != nil { + return err + } + + err = ds.put(bucketFiles, fileKey, fileData) + if err != nil { + return err + } + + return nil +} diff --git a/project.go b/project.go index 89e5640..d1a03ef 100644 --- a/project.go +++ b/project.go @@ -6,6 +6,7 @@ package main import ( "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -145,6 +146,8 @@ func (p *Project) prepData() error { } //TODO: Create data store + + return errors.New("TODO") } type projectList struct { @@ -183,7 +186,7 @@ func (p *projectList) load() error { err = prj.load() if err != nil { - delete(p, files[i].Name()) + delete(p.data, files[i].Name()) return err } }