Go: logrus性能提升

在Go项目中, logrus是一个相对完备的第三方日志库

用起来非常顺手, 特别是WithField/WithFields/WithError

我们开发一些对性能要求非常高的应用, 例如API网关/权限服务等, 需要记录流水日志, 此时日志库的性能直接会影响整体接口性能.

所以针对性地做了一些优化

log.WithFields(fields).Info("-")

json formatter

logrus的默认json formatter json_formatter 使用的是标准库

众所周知, 性能比第三方库差很多

我们可以自定义一个json formatter, 使用例如 json-iterator/go来提升序列化性能

	// ! change here: use jsnoiter to do marshal. but ignore the entry.Buffer

	// var b *bytes.Buffer
	// if entry.Buffer != nil {
	// 	b = entry.Buffer
	// } else {
	// 	b = &bytes.Buffer{}
	// }

	// encoder := json.NewEncoder(b)
	// if f.PrettyPrint {
	// 	encoder.SetIndent("", "  ")
	// }
	// if err := encoder.Encode(data); err != nil {
	// 	return nil, fmt.Errorf("failed to marshal fields to JSON, %v", err)
	// }
	// return b.Bytes(), nil

	buf, err := jsoniter.Marshal(data)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal fields to JSON, %v", err)
	}
	buf = append(buf, '\n')
	return buf, nil

null formatter 和 hooks

如果不指定, 默认的formatter是TextFormatter

func New() *Logger {
	return &Logger{
		Out:          os.Stderr,
		Formatter:    new(TextFormatter),
		Hooks:        make(LevelHooks),
		Level:        InfoLevel,
		ExitFunc:     os.Exit,
		ReportCaller: false,
	}
}

如果我们使用了 hooks (hooks.go)将日志转到文件/redis或其他渠道; 此时, 默认的Output和Formatter还是会被调用到;

其实我们配置了hooks, 那么原entry理论上是不需要的

func (entry Entry) log(level Level, msg string) {
    entry.fireHooks()
    ....
    entry.write()
}

func (entry *Entry) write() {
	entry.Logger.mu.Lock()
	defer entry.Logger.mu.Unlock()
	serialized, err := entry.Logger.Formatter.Format(entry)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to obtain reader, %v\n", err)
		return
	}
	if _, err = entry.Logger.Out.Write(serialized); err != nil {
		fmt.Fprintf(os.Stderr, "Failed to write to log, %v\n", err)
	}
}

所以, 此时可以配置一个 NullFormatter, 将默认的链路关闭(do nothing), 日志只会走hooks

package formatter

import (
	"github.com/sirupsen/logrus"
)

// NullFormatter formats logs into text
type NullFormatter struct {
}

// Format renders a single log entry
func (f *NullFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	return []byte(""), nil
}

重新设置log的配置

logger.SetOutput(ioutil.Discard)
logger.SetFormatter(&formatter.NullFormatter{})

logger.AddHook(NewRedisHook())

log entry pool

package logging

import (
	"sync"

	log "github.com/sirupsen/logrus"
)

type logEntryPool struct {
	pool sync.Pool
}

func newLogEntryPool() *logEntryPool {
	return &logEntryPool{
		pool: sync.Pool{
			New: func() interface{} {
				return &log.Entry{
					// Logger: logger,
					// Default is three fields, plus one optional.  Give a little extra room.
					Data: make(log.Fields, 6),
				}
			},
		},
	}
}

func (p *logEntryPool) Get(logger *log.Logger) *log.Entry {
	entry := p.pool.Get().(*log.Entry)
	entry.Logger = logger
	return entry
}

func (p *logEntryPool) Put(e *log.Entry) {
	// TODO: clean, should make?
	// reference: https://github.com/sirupsen/logrus/pull/796/files
	// e.Data = make(log.Fields, 6)
	e.Data = map[string]interface{}{}

	p.pool.Put(e)
}

使用

entry := logging.LogEntryPool.Get(logger)
defer logging.LogEntryPool.Put(entry)

异步日志

使用一个专门处理日志的goroutine

func (f *FileLogHook) makeAsync() {
	f.fireChannel = make(chan *logrus.Entry, f.asyncBufferSize)
	fmt.Printf("file hook will use a async buffer with size %d\n", f.asyncBufferSize)
	go func() {
		for entry := range f.fireChannel {
			if err := f.send(entry); err != nil {
				fmt.Println("Error during sending message to file:", err)
			}
		}
	}()
}

// Fire is called when a log event is fired.
func (f *FileLogHook) Fire(entry *logrus.Entry) error {
	if f.fireChannel != nil { // Async mode.
		select {
		case f.fireChannel <- entry: // try and put into chan, if fail will to default
		default:
			if f.asyncBlock {
				fmt.Println("the log buffered chan is full! will block")
				f.fireChannel <- entry // Blocks the goroutine because buffer is full.
				return nil
			}
			fmt.Println("the log buffered chan is full! will drop")
			// Drop message by default.
		}
		return nil
	}

  // Sync mode.
	return f.send(entry)
}

func (f *FileLogHook) send(entry *logrus.Entry) error {
	return f.loghook.Fire(entry)
}
Async write to redis 
with buffersize=10000
(block)

7700 -> 18000

Async write to redis
With buffersize=10000
(none block, drop if full)

60000+

golang

918 Words

2021-02-09 20:00 +0800