Showing 5 of 909 files from the diff.
Newly tracked file
pkg/sdb/tx.go created.
Newly tracked file
pkg/sdb/sdb.go created.
Newly tracked file
pkg/sdb/errors.go created.
Newly tracked file
pkg/sdb/backup.go created.
Newly tracked file
pkg/sdb/options.go created.
Other files ignored by Codecov
.travis.yml has changed.
Makefile is new.
mage.go was deleted.
go.mod has changed.
CONTRIBUTING.md has changed.
docs/er.svg was deleted.
go.sum has changed.
pkg/usersd/jwt.go was deleted.
main.go has changed.
pkg/usersd/db.go was deleted.
.gitignore has changed.
magefile.go was deleted.
mage was deleted.
pkg/usersd/doc.go has changed.
Dockerfile has changed.

@@ -0,0 +1,159 @@
Loading
1 +
// Copyright 2019 Miguel Angel Rivera Notararigo. All rights reserved.
2 +
// This source code was released under the MIT license.
3 +
4 +
package sdb
5 +
6 +
import (
7 +
	"encoding/json"
8 +
	"errors"
9 +
10 +
	"github.com/blevesearch/bleve"
11 +
	"github.com/dgraph-io/badger/v2"
12 +
)
13 +
14 +
const (
15 +
	RW = true
16 +
	RO = false
17 +
)
18 +
19 +
// Tx is a transaction object which provides data management methods. The
20 +
// search index doesn't support transactions yet, so indexing operations just
21 +
// take effect after committing the transaction.
22 +
type Tx struct {
23 +
	db   *DB
24 +
	dbTx *badger.Txn
25 +
	si   bleve.Index
26 +
	rw   bool
27 +
28 +
	// Search index operations to be done when the transaction is committed.
29 +
	operations map[string]interface{}
30 +
}
31 +
32 +
// NewTx creates a database transaction. If rw is false, the new transaction
33 +
// will be read-only.
34 +
func (db *DB) NewTx(rw bool) *Tx {
35 +
	return &Tx{
36 +
		db:   db,
37 +
		dbTx: db.db.NewTransaction(rw),
38 +
		si:   db.si,
39 +
		rw:   rw,
40 +
	}
41 +
}
42 +
43 +
// Commit writes the transaction operations to the database. If a Bleve error
44 +
// is returned, the search index should be reloaded (see DB.ReloadIndex), keep
45 +
// the amount of operations per transaction low to avoid this.
46 +
func (tx *Tx) Commit() error {
47 +
	if err := tx.dbTx.Commit(); err != nil {
48 +
		return badgerError(err)
49 +
	}
50 +
51 +
	for id, data := range tx.operations {
52 +
		if data != nil {
53 +
			if err := tx.si.Index(id, data); err != nil {
54 +
				return bleveError(err)
55 +
			}
56 +
		} else {
57 +
			if err := tx.si.Delete(id); err != nil {
58 +
				return bleveError(err)
59 +
			}
60 +
		}
61 +
	}
62 +
63 +
	return nil
64 +
}
65 +
66 +
// Delete deletes the given key. This operation happens in memory, it will be
67 +
// written to the database once Commit is called.
68 +
func (tx *Tx) Delete(key []byte) error {
69 +
	if err := tx.dbTx.Delete(key); err != nil {
70 +
		return badgerError(err)
71 +
	}
72 +
73 +
	if tx.operations == nil {
74 +
		tx.operations = make(map[string]interface{})
75 +
	}
76 +
77 +
	tx.operations[string(key)] = nil
78 +
	return nil
79 +
}
80 +
81 +
// Discard drops all the pending modifications and set the transactions as
82 +
// discarded.
83 +
func (tx *Tx) Discard() {
84 +
	if tx.rw {
85 +
		tx.operations = nil
86 +
	}
87 +
88 +
	tx.dbTx.Discard()
89 +
}
90 +
91 +
// Find fetches the keys from the values that satisfies the given constraints.
92 +
// See http://blevesearch.com/docs/Query-String-Query/ for more info about the
93 +
// the query language syntax. sort is a list of field names used for sorting,
94 +
// any field prefixed by a hyphen (-) will user reverse order.
95 +
func (tx *Tx) Find(q string, sort ...string) ([][]byte, error) {
96 +
	if q == "" && len(sort) == 0 {
97 +
		return nil, nil
98 +
	}
99 +
100 +
	result := [][]byte{}
101 +
	bq := bleve.NewQueryStringQuery(q)
102 +
	req := bleve.NewSearchRequest(bq)
103 +
	req.SortBy(sort)
104 +
105 +
	res, err := tx.si.Search(req)
106 +
	if err != nil {
107 +
		return nil, bleveError(err)
108 +
	}
109 +
110 +
	for _, hit := range res.Hits {
111 +
		result = append(result, []byte(hit.ID))
112 +
	}
113 +
114 +
	return result, nil
115 +
}
116 +
117 +
// Get reads the value from the given key and decodes it into v (must be a
118 +
// pointer).
119 +
func (tx *Tx) Get(key []byte, v interface{}) error {
120 +
	item, err := tx.dbTx.Get(key)
121 +
	if errors.Is(err, badger.ErrKeyNotFound) {
122 +
		return ErrKeyNotFound
123 +
	} else if err != nil {
124 +
		return badgerError(err)
125 +
	}
126 +
127 +
	buf := tx.db.buffers.Get()
128 +
	defer tx.db.buffers.Add(buf)
129 +
130 +
	data, err := item.ValueCopy(buf.Bytes())
131 +
	if err != nil {
132 +
		return err
133 +
	}
134 +
135 +
	return json.Unmarshal(data, v)
136 +
}
137 +
138 +
// Set set val as value of the given key. This operation happens in memory, it
139 +
// will be written to the database once Commit is called.
140 +
func (tx *Tx) Set(key []byte, val interface{}) error {
141 +
	buf := tx.db.buffers.Get()
142 +
	defer tx.db.buffers.Add(buf)
143 +
144 +
	e := json.NewEncoder(buf)
145 +
	if err := e.Encode(val); err != nil {
146 +
		return err
147 +
	}
148 +
149 +
	if err := tx.dbTx.Set(key, buf.Bytes()); err != nil {
150 +
		return badgerError(err)
151 +
	}
152 +
153 +
	if tx.operations == nil {
154 +
		tx.operations = make(map[string]interface{})
155 +
	}
156 +
157 +
	tx.operations[string(key)] = val
158 +
	return nil
159 +
}

@@ -0,0 +1,117 @@
Loading
1 +
// Copyright 2019 Miguel Angel Rivera Notararigo. All rights reserved.
2 +
// This source code was released under the MIT license.
3 +
4 +
package sdb
5 +
6 +
import (
7 +
	"io/ioutil"
8 +
	"log"
9 +
10 +
	"github.com/blevesearch/bleve"
11 +
	"github.com/dgraph-io/badger/v2"
12 +
	"github.com/dgraph-io/badger/v2/options"
13 +
	ntbytes "nt.web.ve/go/ntgo/bytes"
14 +
)
15 +
16 +
// DB is a database object which provides database management methods, for data
17 +
// management see Tx.
18 +
type DB struct {
19 +
	opts Options
20 +
21 +
	db *badger.DB
22 +
	si bleve.Index
23 +
24 +
	buffers *ntbytes.BufferPool
25 +
	logger  *log.Logger
26 +
}
27 +
28 +
// Open initializes a database in the given directory.
29 +
func Open(dir string) (*DB, error) {
30 +
	var err error
31 +
32 +
	if dir == "" {
33 +
		dir, err = ioutil.TempDir("", "sdb")
34 +
		if err != nil {
35 +
			return nil, err
36 +
		}
37 +
	}
38 +
39 +
	opts := DefaultOptions(dir)
40 +
	return OpenWith(opts)
41 +
}
42 +
43 +
// OpenWith initializes a database with the given options.
44 +
func OpenWith(opts Options) (*DB, error) {
45 +
	var err error
46 +
47 +
	db := new(DB)
48 +
49 +
	db.buffers = ntbytes.NewBufferPool(
50 +
		opts.BufferPoolSize,
51 +
		opts.BufferPoolMaxBytes,
52 +
	)
53 +
54 +
	if opts.BufferPoolFill {
55 +
		db.buffers.Fill()
56 +
	}
57 +
58 +
	db.logger = opts.Logger
59 +
60 +
	if db.db, err = openDB(opts.Badger); err != nil {
61 +
		return nil, badgerError(err)
62 +
	}
63 +
64 +
	if db.si, err = openSearchIndex(opts.Bleve); err != nil {
65 +
		return nil, bleveError(err)
66 +
	}
67 +
68 +
	db.opts = opts
69 +
	return db, nil
70 +
}
71 +
72 +
// Close terminates the database.
73 +
func (db *DB) Close() error {
74 +
	if err := db.db.Close(); err != nil {
75 +
		return badgerError(err)
76 +
	}
77 +
78 +
	_, kvs, err := db.si.Advanced()
79 +
	if err != nil {
80 +
		return bleveError(err)
81 +
	}
82 +
83 +
	if err = kvs.Close(); err != nil {
84 +
		return bleveError(err)
85 +
	}
86 +
87 +
	return nil
88 +
}
89 +
90 +
func openDB(opts badger.Options) (*badger.DB, error) {
91 +
	opts.Logger = &bl{}
92 +
	opts.Compression = options.Snappy
93 +
	return badger.Open(opts)
94 +
}
95 +
96 +
func openSearchIndex(opts BleveOptions) (bleve.Index, error) {
97 +
	index, err := bleve.Open(opts.Dir)
98 +
	if err == bleve.Error(1) { // ErrorIndexPathDoesNotExist
99 +
		mapping := bleve.NewIndexMapping()
100 +
		mapping.TypeField = opts.DoctypeField
101 +
102 +
		for t, m := range opts.DocMappings {
103 +
			mapping.AddDocumentMapping(t, m)
104 +
		}
105 +
106 +
		return bleve.New(opts.Dir, mapping)
107 +
	}
108 +
109 +
	return index, err
110 +
}
111 +
112 +
type bl struct{}
113 +
114 +
func (l *bl) Errorf(f string, v ...interface{})   {}
115 +
func (l *bl) Warningf(f string, v ...interface{}) {}
116 +
func (l *bl) Infof(f string, v ...interface{})    {}
117 +
func (l *bl) Debugf(f string, v ...interface{})   {}

@@ -0,0 +1,42 @@
Loading
1 +
// Copyright 2019 Miguel Angel Rivera Notararigo. All rights reserved.
2 +
// This source code was released under the MIT license.
3 +
4 +
package sdb
5 +
6 +
import (
7 +
	"fmt"
8 +
	"strings"
9 +
10 +
	"github.com/dgraph-io/badger/v2"
11 +
)
12 +
13 +
// Errors
14 +
var (
15 +
	ErrKeyNotFound = badgerError(badger.ErrKeyNotFound)
16 +
)
17 +
18 +
// IsBadgerError returns true if the given error is from Badger.
19 +
func IsBadgerError(err error) bool {
20 +
	return errorContains(err, "badger: ")
21 +
}
22 +
23 +
// IsBleveError returns true if the given error is from Bleve.
24 +
func IsBleveError(err error) bool {
25 +
	return errorContains(err, "bleve: ")
26 +
}
27 +
28 +
func badgerError(err error) error {
29 +
	return fmt.Errorf("badger: %w", err)
30 +
}
31 +
32 +
func bleveError(err error) error {
33 +
	return fmt.Errorf("bleve: %w", err)
34 +
}
35 +
36 +
func errorContains(err error, s string) bool {
37 +
	if err == nil {
38 +
		return false
39 +
	}
40 +
41 +
	return strings.Contains(err.Error(), s)
42 +
}

@@ -0,0 +1,52 @@
Loading
1 +
// Copyright 2019 Miguel Angel Rivera Notararigo. All rights reserved.
2 +
// This source code was released under the MIT license.
3 +
4 +
package sdb
5 +
6 +
import (
7 +
	"github.com/blevesearch/bleve"
8 +
	"github.com/dgraph-io/badger/v2"
9 +
)
10 +
11 +
func (db *DB) ReloadIndex() error {
12 +
	if err := db.cleanIndex(); err != nil {
13 +
		return err
14 +
	}
15 +
16 +
	tx := db.NewTx(RO)
17 +
	defer tx.Discard()
18 +
19 +
	it := tx.dbTx.NewIterator(badger.DefaultIteratorOptions)
20 +
	defer it.Close()
21 +
22 +
	for it.Rewind(); it.Valid(); it.Next() {
23 +
		var val interface{}
24 +
		key := it.Item().Key()
25 +
		if err := tx.Get(key, &val); err != nil {
26 +
			return badgerError(err)
27 +
		}
28 +
29 +
		if err := tx.si.Index(string(key), val); err != nil {
30 +
			return bleveError(err)
31 +
		}
32 +
	}
33 +
34 +
	return nil
35 +
}
36 +
37 +
func (db *DB) cleanIndex() error {
38 +
	bq := bleve.NewMatchAllQuery()
39 +
	req := bleve.NewSearchRequest(bq)
40 +
	res, err := db.si.Search(req)
41 +
	if err != nil {
42 +
		return bleveError(err)
43 +
	}
44 +
45 +
	for _, hit := range res.Hits {
46 +
		if err = db.si.Delete(hit.ID); err != nil {
47 +
			return bleveError(err)
48 +
		}
49 +
	}
50 +
51 +
	return nil
52 +
}

@@ -0,0 +1,61 @@
Loading
1 +
// Copyright 2019 Miguel Angel Rivera Notararigo. All rights reserved.
2 +
// This source code was released under the MIT license.
3 +
4 +
package sdb
5 +
6 +
import (
7 +
	"log"
8 +
	"os"
9 +
	"path/filepath"
10 +
11 +
	"github.com/blevesearch/bleve/mapping"
12 +
	"github.com/dgraph-io/badger/v2"
13 +
)
14 +
15 +
const (
16 +
	DatabaseDir    = "database"
17 +
	SearchIndexDir = "search-index"
18 +
)
19 +
20 +
type BleveOptions struct {
21 +
	Dir          string
22 +
	DoctypeField string
23 +
	DocMappings  map[string]*mapping.DocumentMapping
24 +
}
25 +
26 +
// Options are parameters for initializing a database.
27 +
type Options struct {
28 +
	// Database location.
29 +
	Directory string
30 +
31 +
	Badger badger.Options
32 +
	Bleve  BleveOptions
33 +
34 +
	BufferPoolSize     int  // Amount of buffers.
35 +
	BufferPoolMaxBytes int  // Bytes limit per buffer.
36 +
	BufferPoolFill     bool // Fill up the pool at DB creation.
37 +
38 +
	Logger *log.Logger
39 +
}
40 +
41 +
// DefaultOptions returns commonly used options for creating a database.
42 +
func DefaultOptions(dir string) Options {
43 +
	dir = filepath.Clean(dir)
44 +
45 +
	return Options{
46 +
		Directory: dir,
47 +
		Badger:    badger.DefaultOptions(filepath.Join(dir, DatabaseDir)),
48 +
49 +
		Bleve: BleveOptions{
50 +
			Dir:          filepath.Join(dir, SearchIndexDir),
51 +
			DoctypeField: "Doctype",
52 +
			DocMappings:  make(map[string]*mapping.DocumentMapping),
53 +
		},
54 +
55 +
		BufferPoolSize:     500,
56 +
		BufferPoolMaxBytes: 5 * 1024,
57 +
		BufferPoolFill:     false,
58 +
59 +
		Logger: log.New(os.Stderr, "sdb: ", log.LstdFlags),
60 +
	}
61 +
}
Files Coverage
pkg/sdb 77.11%
Project Totals (5 files) 77.11%
35.4
TRAVIS_OS_NAME=linux
1.13=.13

No yaml found.

Create your codecov.yml to customize your Codecov experience

Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading