Added datastore handling

Started on basic data layer stuff
This commit is contained in:
Tim Shannon 2016-03-31 21:59:48 +00:00
parent cb883bc528
commit a5660ba38f
6 changed files with 325 additions and 1 deletions

View File

@ -4,3 +4,111 @@
// Package datastore manages the bolt data files and the reading and writing data to them // Package datastore manages the bolt data files and the reading and writing data to them
package datastore package datastore
import (
"bytes"
"encoding/json"
"errors"
"io"
"time"
"github.com/boltdb/bolt"
)
// ErrNotFound is the error returned when a value cannot be found in the store for the given key
var ErrNotFound = errors.New("Value not found")
// Store is a datastore for getting and setting data for a given ironsmith project
// run on top of a Bolt DB file
type Store struct {
bolt *bolt.DB
}
// Open opens an existing datastore file, or creates a new one
// caller is responsible for closing the datastore
func Open(filename string) (*Store, error) {
db, err := bolt.Open(filename, 0666, &bolt.Options{Timeout: 1 * time.Minute})
if err != nil {
return nil, err
}
store := &Store{
bolt: db,
}
err = store.bolt.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(bucketLog))
if err != nil {
return err
}
_, err = tx.CreateBucketIfNotExists([]byte(bucketReleases))
if err != nil {
return err
}
_, err = tx.CreateBucketIfNotExists([]byte(bucketFiles))
if err != nil {
return err
}
return nil
})
if err != nil {
return nil, err
}
return store, nil
}
// Close closes the bolt datastore
func (ds *Store) Close() error {
if ds != nil {
return ds.Close()
}
return nil
}
func (ds *Store) get(bucket string, key TimeKey, result interface{}) error {
return ds.bolt.View(func(tx *bolt.Tx) error {
dsValue := tx.Bucket([]byte(bucket)).Get(key.Bytes())
if dsValue == nil {
return ErrNotFound
}
if value, ok := result.([]byte); ok {
buff := bytes.NewBuffer(value)
_, err := io.Copy(buff, bytes.NewReader(dsValue))
if err != nil {
return err
}
return nil
}
return json.Unmarshal(dsValue, result)
})
}
func (ds *Store) put(bucket string, key TimeKey, value interface{}) error {
var err error
dsValue, ok := value.([]byte)
if !ok {
dsValue, err = json.Marshal(value)
if err != nil {
return err
}
}
return ds.bolt.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(bucket)).Put(key.Bytes(), dsValue)
})
}
func (ds *Store) delete(bucket string, key TimeKey) error {
return ds.bolt.Update(func(tx *bolt.Tx) error {
return tx.Bucket([]byte(bucket)).Delete(key.Bytes())
})
}

56
datastore/ds_test.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package datastore
import (
"sort"
"testing"
"time"
)
func TestNewTimeKey(t *testing.T) {
tk := NewTimeKey()
if len(tk) != (128 / 8) {
t.Errorf("Invalid Time Key Length want %d got %d", 128/8, len(tk))
}
}
func TestTimeKeyTime(t *testing.T) {
now := time.Now()
tk := NewTimeKey()
tkTime := tk.Time()
if !tkTime.After(now) && !tkTime.Equal(now) {
t.Errorf("TimeKey's time is not after or equal a previous generated timestamp. want: %s, got: %s", now, tkTime)
}
cTime := tk.Time()
if !tkTime.Equal(cTime) {
t.Errorf("TimeKey's time is not consistently parsed from the timekey. want: %s, got: %s", tkTime, cTime)
}
}
type ByTime []TimeKey
func (a ByTime) Len() int { return len(a) }
func (a ByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByTime) Less(i, j int) bool { return a[i].Time().Before(a[j].Time()) }
func TestTimeKeyOrder(t *testing.T) {
keys := make([]TimeKey, 1000)
for i := range keys {
keys[i] = NewTimeKey()
}
if !sort.IsSorted(ByTime(keys)) {
t.Errorf("TimeKey's are not properly sorted by time")
}
}

85
datastore/key.go Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package datastore
import (
"crypto/rand"
"encoding/hex"
"fmt"
"io"
"time"
)
// TimeKey is a unique time ordered key for use in the datastore
// A TimeKey is 32 bits random data + 96 bit UNIX timestamp (64bits seconds + 32 bit nanoseconds)
type TimeKey [16]byte
// NewTimeKey returns a newly generated TimeKey based on the current time
func NewTimeKey() TimeKey {
rBits := make([]byte, 32/8)
_, err := io.ReadFull(rand.Reader, rBits)
if err != nil {
panic(fmt.Sprintf("Error generating random values for New TimeKey: %v", err))
}
t := time.Now()
sec := t.Unix()
nsec := t.Nanosecond()
return TimeKey{
rBits[0], //random
rBits[1],
rBits[2],
rBits[3],
byte(sec >> 56), // seconds
byte(sec >> 48),
byte(sec >> 40),
byte(sec >> 32),
byte(sec >> 24),
byte(sec >> 16),
byte(sec >> 8),
byte(sec),
byte(nsec >> 24), // nanoseconds
byte(nsec >> 16),
byte(nsec >> 8),
byte(nsec),
}
}
// Time returns the time portion of a timekey
func (k TimeKey) Time() time.Time {
buf := k[4:]
sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 |
int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56
buf = buf[8:]
nsec := int32(buf[3]) | int32(buf[2])<<8 | int32(buf[1])<<16 | int32(buf[0])<<24
return time.Unix(sec, int64(nsec))
}
// UUID returns the string representation of a TimeKey in Hex format separated by dashes
// similar to a UUID xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
func (k TimeKey) UUID() string {
buf := make([]byte, 36)
hex.Encode(buf[0:8], k[0:4])
buf[8] = '-'
hex.Encode(buf[9:13], k[4:6])
buf[13] = '-'
hex.Encode(buf[14:18], k[6:8])
buf[18] = '-'
hex.Encode(buf[19:23], k[8:10])
buf[23] = '-'
hex.Encode(buf[24:], k[10:])
return string(buf)
}
// Bytes returns the a slice of the underlying bytes of a TimeKey
func (k TimeKey) Bytes() []byte {
return []byte(k[:])
}

30
datastore/log.go Normal file
View File

@ -0,0 +1,30 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package datastore
import "time"
type log struct {
When time.Time `json:"when"`
Version string `json:"version"`
Stage string `json:"stage"`
Log string `json:"log"`
}
const bucketLog = "log"
// AddLog adds a new log entry
func (ds *Store) AddLog(version, stage, entry string) error {
key := NewTimeKey()
data := &log{
When: key.Time(),
Version: version,
Stage: stage,
Log: entry,
}
return ds.put(bucketLog, key, data)
}

42
datastore/releases.go Normal file
View File

@ -0,0 +1,42 @@
// Copyright 2016 Tim Shannon. All rights reserved.
// Use of this source code is governed by the MIT license
// that can be found in the LICENSE file.
package datastore
import "time"
type release struct {
When time.Time `json:"when"`
Version string `json:"version"`
FileKey TimeKey `json:"file"`
}
const (
bucketReleases = "releases"
bucketFiles = "files"
)
// AddRelease adds a new Release
func (ds *Store) AddRelease(version string, fileData []byte) error {
key := NewTimeKey()
fileKey := NewTimeKey()
r := &release{
When: key.Time(),
Version: version,
FileKey: fileKey,
}
err := ds.put(bucketReleases, key, r)
if err != nil {
return err
}
err = ds.put(bucketFiles, fileKey, fileData)
if err != nil {
return err
}
return nil
}

View File

@ -6,6 +6,7 @@ package main
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
@ -145,6 +146,8 @@ func (p *Project) prepData() error {
} }
//TODO: Create data store //TODO: Create data store
return errors.New("TODO")
} }
type projectList struct { type projectList struct {
@ -183,7 +186,7 @@ func (p *projectList) load() error {
err = prj.load() err = prj.load()
if err != nil { if err != nil {
delete(p, files[i].Name()) delete(p.data, files[i].Name())
return err return err
} }
} }