Added datastore handling

Started on basic data layer stuff

datastore/ds.go

@@ -4,3 +4,111 @@
 
 // Package datastore manages the bolt data files and the reading and writing data to them
 package datastore
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io"
+	"time"
+
+	"github.com/boltdb/bolt"
+)
+
+// ErrNotFound is the error returned when a value cannot be found in the store for the given key
+var ErrNotFound = errors.New("Value not found")
+
+// Store is a datastore for getting and setting data for a given ironsmith project
+// run on top of a Bolt DB file
+type Store struct {
+	bolt *bolt.DB
+}
+
+// Open opens an existing datastore file, or creates a new one
+// caller is responsible for closing the datastore
+func Open(filename string) (*Store, error) {
+	db, err := bolt.Open(filename, 0666, &bolt.Options{Timeout: 1 * time.Minute})
+
+	if err != nil {
+		return nil, err
+	}
+
+	store := &Store{
+		bolt: db,
+	}
+
+	err = store.bolt.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(bucketLog))
+		if err != nil {
+			return err
+		}
+		_, err = tx.CreateBucketIfNotExists([]byte(bucketReleases))
+		if err != nil {
+			return err
+		}
+
+		_, err = tx.CreateBucketIfNotExists([]byte(bucketFiles))
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return store, nil
+}
+
+// Close closes the bolt datastore
+func (ds *Store) Close() error {
+	if ds != nil {
+		return ds.Close()
+	}
+
+	return nil
+}
+
+func (ds *Store) get(bucket string, key TimeKey, result interface{}) error {
+	return ds.bolt.View(func(tx *bolt.Tx) error {
+		dsValue := tx.Bucket([]byte(bucket)).Get(key.Bytes())
+
+		if dsValue == nil {
+			return ErrNotFound
+		}
+
+		if value, ok := result.([]byte); ok {
+			buff := bytes.NewBuffer(value)
+			_, err := io.Copy(buff, bytes.NewReader(dsValue))
+			if err != nil {
+				return err
+			}
+			return nil
+		}
+
+		return json.Unmarshal(dsValue, result)
+	})
+}
+
+func (ds *Store) put(bucket string, key TimeKey, value interface{}) error {
+	var err error
+	dsValue, ok := value.([]byte)
+	if !ok {
+		dsValue, err = json.Marshal(value)
+		if err != nil {
+			return err
+		}
+	}
+
+	return ds.bolt.Update(func(tx *bolt.Tx) error {
+		return tx.Bucket([]byte(bucket)).Put(key.Bytes(), dsValue)
+	})
+}
+
+func (ds *Store) delete(bucket string, key TimeKey) error {
+	return ds.bolt.Update(func(tx *bolt.Tx) error {
+		return tx.Bucket([]byte(bucket)).Delete(key.Bytes())
+	})
+}

datastore/ds_test.go

@@ -0,0 +1,56 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package datastore
+
+import (
+	"sort"
+	"testing"
+	"time"
+)
+
+func TestNewTimeKey(t *testing.T) {
+	tk := NewTimeKey()
+
+	if len(tk) != (128 / 8) {
+		t.Errorf("Invalid Time Key Length want %d got %d", 128/8, len(tk))
+	}
+}
+
+func TestTimeKeyTime(t *testing.T) {
+
+	now := time.Now()
+
+	tk := NewTimeKey()
+
+	tkTime := tk.Time()
+
+	if !tkTime.After(now) && !tkTime.Equal(now) {
+		t.Errorf("TimeKey's time is not after or equal a previous generated timestamp. want: %s, got: %s", now, tkTime)
+	}
+
+	cTime := tk.Time()
+
+	if !tkTime.Equal(cTime) {
+		t.Errorf("TimeKey's time is not consistently parsed from the timekey.  want: %s, got: %s", tkTime, cTime)
+	}
+}
+
+type ByTime []TimeKey
+
+func (a ByTime) Len() int           { return len(a) }
+func (a ByTime) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a ByTime) Less(i, j int) bool { return a[i].Time().Before(a[j].Time()) }
+
+func TestTimeKeyOrder(t *testing.T) {
+	keys := make([]TimeKey, 1000)
+	for i := range keys {
+		keys[i] = NewTimeKey()
+	}
+
+	if !sort.IsSorted(ByTime(keys)) {
+		t.Errorf("TimeKey's are not properly sorted by time")
+	}
+
+}

datastore/key.go

@@ -0,0 +1,85 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package datastore
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"time"
+)
+
+// TimeKey is a unique time ordered key for use in the datastore
+// A TimeKey is 32 bits random data + 96 bit UNIX timestamp (64bits seconds + 32 bit nanoseconds)
+type TimeKey [16]byte
+
+// NewTimeKey returns a newly generated TimeKey based on the current time
+func NewTimeKey() TimeKey {
+	rBits := make([]byte, 32/8)
+	_, err := io.ReadFull(rand.Reader, rBits)
+	if err != nil {
+		panic(fmt.Sprintf("Error generating random values for New TimeKey: %v", err))
+	}
+
+	t := time.Now()
+	sec := t.Unix()
+	nsec := t.Nanosecond()
+
+	return TimeKey{
+		rBits[0], //random
+		rBits[1],
+		rBits[2],
+		rBits[3],
+		byte(sec >> 56), // seconds
+		byte(sec >> 48),
+		byte(sec >> 40),
+		byte(sec >> 32),
+		byte(sec >> 24),
+		byte(sec >> 16),
+		byte(sec >> 8),
+		byte(sec),
+		byte(nsec >> 24), // nanoseconds
+		byte(nsec >> 16),
+		byte(nsec >> 8),
+		byte(nsec),
+	}
+}
+
+// Time returns the time portion of a timekey
+func (k TimeKey) Time() time.Time {
+	buf := k[4:]
+
+	sec := int64(buf[7]) | int64(buf[6])<<8 | int64(buf[5])<<16 | int64(buf[4])<<24 |
+		int64(buf[3])<<32 | int64(buf[2])<<40 | int64(buf[1])<<48 | int64(buf[0])<<56
+
+	buf = buf[8:]
+	nsec := int32(buf[3]) | int32(buf[2])<<8 | int32(buf[1])<<16 | int32(buf[0])<<24
+
+	return time.Unix(sec, int64(nsec))
+}
+
+// UUID returns the string representation of a TimeKey in Hex format separated by dashes
+// similar to a UUID xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+func (k TimeKey) UUID() string {
+	buf := make([]byte, 36)
+
+	hex.Encode(buf[0:8], k[0:4])
+	buf[8] = '-'
+	hex.Encode(buf[9:13], k[4:6])
+	buf[13] = '-'
+	hex.Encode(buf[14:18], k[6:8])
+	buf[18] = '-'
+	hex.Encode(buf[19:23], k[8:10])
+	buf[23] = '-'
+	hex.Encode(buf[24:], k[10:])
+
+	return string(buf)
+}
+
+// Bytes returns the a slice of the underlying bytes of a TimeKey
+func (k TimeKey) Bytes() []byte {
+	return []byte(k[:])
+}

datastore/log.go

@@ -0,0 +1,30 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package datastore
+
+import "time"
+
+type log struct {
+	When    time.Time `json:"when"`
+	Version string    `json:"version"`
+	Stage   string    `json:"stage"`
+	Log     string    `json:"log"`
+}
+
+const bucketLog = "log"
+
+// AddLog adds a new log entry
+func (ds *Store) AddLog(version, stage, entry string) error {
+	key := NewTimeKey()
+
+	data := &log{
+		When:    key.Time(),
+		Version: version,
+		Stage:   stage,
+		Log:     entry,
+	}
+
+	return ds.put(bucketLog, key, data)
+}

datastore/releases.go

@@ -0,0 +1,42 @@
+// Copyright 2016 Tim Shannon. All rights reserved.
+// Use of this source code is governed by the MIT license
+// that can be found in the LICENSE file.
+
+package datastore
+
+import "time"
+
+type release struct {
+	When    time.Time `json:"when"`
+	Version string    `json:"version"`
+	FileKey TimeKey   `json:"file"`
+}
+
+const (
+	bucketReleases = "releases"
+	bucketFiles    = "files"
+)
+
+// AddRelease adds a new Release
+func (ds *Store) AddRelease(version string, fileData []byte) error {
+	key := NewTimeKey()
+	fileKey := NewTimeKey()
+
+	r := &release{
+		When:    key.Time(),
+		Version: version,
+		FileKey: fileKey,
+	}
+
+	err := ds.put(bucketReleases, key, r)
+	if err != nil {
+		return err
+	}
+
+	err = ds.put(bucketFiles, fileKey, fileData)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

project.go

@@ -6,6 +6,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -145,6 +146,8 @@ func (p *Project) prepData() error {
 	}
 
 	//TODO: Create data store
+
+	return errors.New("TODO")
 }
 
 type projectList struct {
@@ -183,7 +186,7 @@ func (p *projectList) load() error {
 
 			err = prj.load()
 			if err != nil {
-				delete(p, files[i].Name())
+				delete(p.data, files[i].Name())
 				return err
 			}
 		}