qiniu / logkit

@@ -1,17 +1,14 @@
Loading
1 1
package mutate
2 2
3 3
import (
4 -
	"bytes"
5 4
	"errors"
6 -
	"strconv"
5 +
	"fmt"
7 6
	"strings"
8 7
	"sync"
9 8
10 -
	"github.com/go-logfmt/logfmt"
11 -
12 -
	"github.com/qiniu/log"
13 9
	"github.com/qiniu/logkit/transforms"
14 10
	. "github.com/qiniu/logkit/utils/models"
11 +
	"github.com/qiniu/logkit/utils/parse/mutate"
15 12
)
16 13
17 14
var (
@@ -254,55 +251,18 @@
Loading
254 251
	wg.Done()
255 252
}
256 253
254 +
// 弃用logfmt切分方式
257 255
func kvTransform(strVal string, splitter string, keepString bool) (Data, error) {
258 -
	var (
259 -
		reader  = bytes.NewReader([]byte(strVal))
260 -
		decoder = logfmt.NewDecoder(reader)
261 -
		data    = make(Data, 0)
262 -
		fields  Data
263 -
	)
264 -
	for {
265 -
		ok := decoder.ScanRecord()
266 -
		if !ok {
267 -
			err := decoder.Err()
268 -
			if err != nil {
269 -
				return nil, err
270 -
			}
271 -
			//此错误仅用于当原始数据解析成功但无解析数据时,保留原始数据之用
272 -
			if len(fields) == 0 {
273 -
				log.Error("no value was parsed after logfmt, will keep origin data in pandora_stash if disable_record_errdata field is false")
274 -
				break
275 -
			}
276 -
			break
277 -
		}
278 -
		fields = make(Data)
279 -
		for decoder.ScanKeyval(splitter[0]) {
280 -
			if string(decoder.Value()) == "" {
281 -
				continue
282 -
			}
283 -
			//type conversions
284 -
			value := string(decoder.Value())
285 -
			if !keepString {
286 -
				if fValue, err := strconv.ParseFloat(value, 64); err == nil {
287 -
					fields[string(decoder.Key())] = fValue
288 -
					continue
289 -
				}
290 -
			}
291 -
			if bValue, err := strconv.ParseBool(value); err == nil {
292 -
				fields[string(decoder.Key())] = bValue
293 -
				continue
294 -
			}
295 -
			fields[string(decoder.Key())] = value
296 -
		}
297 -
		if len(fields) == 0 {
298 -
			continue
299 -
		}
256 +
	mp := mutate.Parser{
257 +
		KeepString: keepString,
258 +
		Splitter:   splitter,
259 +
	}
300 260
301 -
		for fieldKey, fieldVal := range fields {
302 -
			data[fieldKey] = fieldVal
303 -
		}
261 +
	datas, err := mp.Parse(strVal)
262 +
	if err != nil {
263 +
		return nil, fmt.Errorf("parse transform key value failed, error msg: %s", err.Error())
304 264
	}
305 -
	return data, nil
265 +
	return datas[0], nil
306 266
}
307 267
308 268
func (k *KV) delete(curData Data, errNum int, err error) (int, error) {

@@ -1,21 +1,14 @@
Loading
1 1
package logfmt
2 2
3 3
import (
4 -
	"errors"
5 4
	"fmt"
6 -
	"strconv"
7 -
	"strings"
8 5
	"sync"
9 -
	"unicode"
10 6
11 7
	"github.com/qiniu/logkit/conf"
12 8
	"github.com/qiniu/logkit/parser"
13 9
	. "github.com/qiniu/logkit/parser/config"
14 10
	. "github.com/qiniu/logkit/utils/models"
15 -
)
16 -
17 -
const (
18 -
	errMsg = "will keep origin data in pandora_stash if disable_record_errdata field is false"
11 +
	"github.com/qiniu/logkit/utils/parse/mutate"
19 12
)
20 13
21 14
func init() {
@@ -53,8 +46,12 @@
Loading
53 46
}
54 47
55 48
func (p *Parser) Parse(lines []string) ([]Data, error) {
56 -
	if p.splitter == "" {
57 -
		p.splitter = "="
49 +
	mp := mutate.Parser{
50 +
		KeepString: p.keepString,
51 +
		Splitter:   p.splitter,
52 +
	}
53 +
	if mp.Splitter == "" {
54 +
		mp.Splitter = "="
58 55
	}
59 56
	var (
60 57
		lineLen = len(lines)
@@ -73,7 +70,7 @@
Loading
73 70
74 71
	for i := 0; i < numRoutine; i++ {
75 72
		wg.Add(1)
76 -
		go parser.ParseLineDataSlice(sendChan, resultChan, wg, true, p.parse)
73 +
		go parser.ParseLineDataSlice(sendChan, resultChan, wg, true, mp.Parse)
77 74
	}
78 75
79 76
	go func() {
@@ -146,118 +143,6 @@
Loading
146 143
	return datas, se
147 144
}
148 145
149 -
func (p *Parser) parse(line string) ([]Data, error) {
150 -
151 -
	pairs, err := splitKV(line, p.splitter)
152 -
	if err != nil {
153 -
		return nil, err
154 -
	}
155 -
156 -
	// 调整数据类型
157 -
	if len(pairs)%2 == 1 {
158 -
		return nil, errors.New(fmt.Sprintf("key value not match, %s", errMsg))
159 -
	}
160 -
161 -
	data := make([]Data, 0, 1)
162 -
	field := make(Data)
163 -
	for i := 0; i < len(pairs); i += 2 {
164 -
		// 消除双引号; 针对foo="" ,"foo=" 情况;其他情况如 a"b"c=d"e"f等首尾不出现引号的情况视作合法。
165 -
		kNum := strings.Count(pairs[i], "\"")
166 -
		vNum := strings.Count(pairs[i+1], "\"")
167 -
		if kNum%2 == 1 && vNum%2 == 1 {
168 -
			if strings.HasPrefix(pairs[i], "\"") && strings.HasSuffix(pairs[i+1], "\"") {
169 -
				pairs[i] = pairs[i][1:]
170 -
				pairs[i+1] = pairs[i+1][:len(pairs[i+1])-1]
171 -
			}
172 -
		}
173 -
		if kNum%2 == 0 && len(pairs[i]) > 1 {
174 -
			if strings.HasPrefix(pairs[i], "\"") && strings.HasSuffix(pairs[i], "\"") {
175 -
				pairs[i] = pairs[i][1 : len(pairs[i])-1]
176 -
			}
177 -
		}
178 -
		if vNum%2 == 0 && len(pairs[i+1]) > 1 {
179 -
			if strings.HasPrefix(pairs[i+1], "\"") && strings.HasSuffix(pairs[i+1], "\"") {
180 -
				pairs[i+1] = pairs[i+1][1 : len(pairs[i+1])-1]
181 -
			}
182 -
		}
183 -
184 -
		if len(pairs[i]) == 0 || len(pairs[i+1]) == 0 {
185 -
			return nil, fmt.Errorf("no value or key was parsed after logfmt, %s", errMsg)
186 -
		}
187 -
188 -
		value := pairs[i+1]
189 -
		if !p.keepString {
190 -
			if fValue, err := strconv.ParseFloat(value, 64); err == nil {
191 -
				field[pairs[i]] = fValue
192 -
				continue
193 -
			}
194 -
			if bValue, err := strconv.ParseBool(value); err == nil {
195 -
				field[pairs[i]] = bValue
196 -
				continue
197 -
			}
198 -
199 -
		}
200 -
		field[pairs[i]] = value
201 -
	}
202 -
	if len(field) == 0 {
203 -
		return nil, fmt.Errorf("data is empty after parse, %s", errMsg)
204 -
	}
205 -
206 -
	data = append(data, field)
207 -
	return data, nil
208 -
}
209 -
210 -
func splitKV(line string, sep string) ([]string, error) {
211 -
	data := make([]string, 0, 100)
212 -
213 -
	if !strings.Contains(line, sep) {
214 -
		return nil, errors.New(fmt.Sprintf("no splitter exist, %s", errMsg))
215 -
	}
216 -
217 -
	kvArr := make([]string, 0, 100)
218 -
	isKey := true
219 -
	vhead := 0
220 -
	lastSpace := 0
221 -
	pos := 0
222 -
	sepLen := len(sep)
223 -
224 -
	// key或value值中包含sep的情况;默认key中不包含sep;导致algorithm = 1+1=2会变成合法
225 -
	for pos+sepLen <= len(line) {
226 -
		if unicode.IsSpace(rune(line[pos : pos+1][0])) {
227 -
			nextSep := strings.Index(line[pos+1:], sep)
228 -
			if nextSep == -1 {
229 -
				break
230 -
			}
231 -
			if strings.TrimSpace(line[pos+1:pos+1+nextSep]) != "" {
232 -
				lastSpace = pos
233 -
				pos++
234 -
				continue
235 -
			}
236 -
		}
237 -
		if line[pos:pos+sepLen] == sep {
238 -
			if isKey {
239 -
				kvArr = append(kvArr, strings.TrimSpace(line[vhead:pos]))
240 -
				isKey = false
241 -
			} else {
242 -
				if lastSpace <= vhead {
243 -
					pos++
244 -
					continue
245 -
				}
246 -
				kvArr = append(kvArr, strings.TrimSpace(line[vhead:lastSpace]))
247 -
				kvArr = append(kvArr, strings.TrimSpace(line[lastSpace:pos]))
248 -
			}
249 -
			vhead = pos + sepLen
250 -
			pos = pos + sepLen - 1
251 -
		}
252 -
		pos++
253 -
	}
254 -
	if vhead < len(line) {
255 -
		kvArr = append(kvArr, strings.TrimSpace(line[vhead:]))
256 -
	}
257 -
	data = append(data, kvArr...)
258 -
	return data, nil
259 -
}
260 -
261 146
func (p *Parser) Name() string {
262 147
	return p.name
263 148
}

@@ -0,0 +1,132 @@
Loading
1 +
package mutate
2 +
3 +
import (
4 +
	"errors"
5 +
	"fmt"
6 +
	"strconv"
7 +
	"strings"
8 +
	"unicode"
9 +
10 +
	"github.com/qiniu/logkit/utils/models"
11 +
)
12 +
13 +
const (
14 +
	errMsg = "will keep origin data in pandora_stash if disable_record_errdata field is false"
15 +
)
16 +
17 +
type Parser struct {
18 +
	KeepString bool
19 +
	Splitter   string
20 +
}
21 +
22 +
func (p *Parser) Parse(line string) ([]models.Data, error) {
23 +
24 +
	pairs, err := splitKV(line, p.Splitter)
25 +
	if err != nil {
26 +
		return nil, err
27 +
	}
28 +
29 +
	// 调整数据类型
30 +
	if len(pairs)%2 == 1 {
31 +
		return nil, errors.New(fmt.Sprintf("key value not match, %s", errMsg))
32 +
	}
33 +
34 +
	data := make([]models.Data, 0, 1)
35 +
	field := make(models.Data)
36 +
	for i := 0; i < len(pairs); i += 2 {
37 +
		// 消除双引号; 针对foo="" ,"foo=" 情况;其他情况如 a"b"c=d"e"f等首尾不出现引号的情况视作合法。
38 +
		kNum := strings.Count(pairs[i], "\"")
39 +
		vNum := strings.Count(pairs[i+1], "\"")
40 +
		if kNum%2 == 1 && vNum%2 == 1 {
41 +
			if strings.HasPrefix(pairs[i], "\"") && strings.HasSuffix(pairs[i+1], "\"") {
42 +
				pairs[i] = pairs[i][1:]
43 +
				pairs[i+1] = pairs[i+1][:len(pairs[i+1])-1]
44 +
			}
45 +
		}
46 +
		if kNum%2 == 0 && len(pairs[i]) > 1 {
47 +
			if strings.HasPrefix(pairs[i], "\"") && strings.HasSuffix(pairs[i], "\"") {
48 +
				pairs[i] = pairs[i][1 : len(pairs[i])-1]
49 +
			}
50 +
		}
51 +
		if vNum%2 == 0 && len(pairs[i+1]) > 1 {
52 +
			if strings.HasPrefix(pairs[i+1], "\"") && strings.HasSuffix(pairs[i+1], "\"") {
53 +
				pairs[i+1] = pairs[i+1][1 : len(pairs[i+1])-1]
54 +
			}
55 +
		}
56 +
57 +
		if len(pairs[i]) == 0 || len(pairs[i+1]) == 0 {
58 +
			return nil, fmt.Errorf("no value or key was parsed after logfmt, %s", errMsg)
59 +
		}
60 +
61 +
		value := pairs[i+1]
62 +
		if !p.KeepString {
63 +
			if fValue, err := strconv.ParseFloat(value, 64); err == nil {
64 +
				field[pairs[i]] = fValue
65 +
				continue
66 +
			}
67 +
			if bValue, err := strconv.ParseBool(value); err == nil {
68 +
				field[pairs[i]] = bValue
69 +
				continue
70 +
			}
71 +
72 +
		}
73 +
		field[pairs[i]] = value
74 +
	}
75 +
	if len(field) == 0 {
76 +
		return nil, fmt.Errorf("data is empty after parse, %s", errMsg)
77 +
	}
78 +
79 +
	data = append(data, field)
80 +
	return data, nil
81 +
}
82 +
83 +
func splitKV(line string, sep string) ([]string, error) {
84 +
	data := make([]string, 0, 100)
85 +
86 +
	if !strings.Contains(line, sep) {
87 +
		return nil, errors.New(fmt.Sprintf("no splitter exist, %s", errMsg))
88 +
	}
89 +
90 +
	kvArr := make([]string, 0, 100)
91 +
	isKey := true
92 +
	vhead := 0
93 +
	lastSpace := 0
94 +
	pos := 0
95 +
	sepLen := len(sep)
96 +
97 +
	// key或value值中包含sep的情况;默认key中不包含sep;导致algorithm = 1+1=2会变成合法
98 +
	for pos+sepLen <= len(line) {
99 +
		if unicode.IsSpace(rune(line[pos : pos+1][0])) {
100 +
			nextSep := strings.Index(line[pos+1:], sep)
101 +
			if nextSep == -1 {
102 +
				break
103 +
			}
104 +
			if strings.TrimSpace(line[pos+1:pos+1+nextSep]) != "" {
105 +
				lastSpace = pos
106 +
				pos++
107 +
				continue
108 +
			}
109 +
		}
110 +
		if line[pos:pos+sepLen] == sep {
111 +
			if isKey {
112 +
				kvArr = append(kvArr, strings.TrimSpace(line[vhead:pos]))
113 +
				isKey = false
114 +
			} else {
115 +
				if lastSpace <= vhead {
116 +
					pos++
117 +
					continue
118 +
				}
119 +
				kvArr = append(kvArr, strings.TrimSpace(line[vhead:lastSpace]))
120 +
				kvArr = append(kvArr, strings.TrimSpace(line[lastSpace:pos]))
121 +
			}
122 +
			vhead = pos + sepLen
123 +
			pos = pos + sepLen - 1
124 +
		}
125 +
		pos++
126 +
	}
127 +
	if vhead < len(line) {
128 +
		kvArr = append(kvArr, strings.TrimSpace(line[vhead:]))
129 +
	}
130 +
	data = append(data, kvArr...)
131 +
	return data, nil
132 +
}
Files Coverage
conf 97.67%
mgr 66.28%
parser 87.12%
queue 74.13%
rateio 95.74%
reader 47.88%
router 54.13%
samples 100.00%
sender 45.15%
transforms 55.67%
utils 58.88%
audit/audit.go 73.63%
cleaner/cleaner.go 58.82%
cli/upgrade.go 47.56%
logkit.go 35.20%
self/logrunner.go 53.88%
times/times.go 91.30%
Project Totals (139 files) 57.03%
3964.1
TRAVIS_OS_NAME=linux
1.12.9=.12.9

No yaml found.

Create your codecov.yml to customize your Codecov experience

Sunburst
The inner-most circle is the entire project, moving away from the center are folders then, finally, a single file. The size and color of each slice is representing the number of statements and the coverage, respectively.
Icicle
The top section represents the entire project. Proceeding with folders and finally individual files. The size and color of each slice is representing the number of statements and the coverage, respectively.
Grid
Each block represents a single file in the project. The size and color of each block is represented by the number of statements and the coverage, respectively.
Loading