Compare commits

..

12 Commits

Author SHA1 Message Date
8b94a0d256 修改es账号密码 2022-10-31 15:41:38 +08:00
38247cf3c9 修复es type错误问题 2022-10-28 17:18:50 +08:00
36a085eccb 修改es服务端地址 2022-10-28 16:03:47 +08:00
liangzy
e0682ed122 v1.3.5 Fix 修复goroutine并发调用初始化 nil pointer的问题 2020-05-16 14:45:18 +08:00
liangzy
cd85fd66ff 并发读nil错针的问题 2020-05-15 18:33:47 +08:00
liangzy
3c03b5c497 v1.3.3 暴露两个配置方法 2020-05-07 10:59:16 +08:00
liangzy
cd94900f19 v1.3.3 暴露两个配置方法 2020-05-06 15:34:47 +08:00
liangzy
ce9ab060d9 v1.3.3 暴露两个配置方法 2020-05-06 15:30:24 +08:00
liangzy
c653a48667 v1.3.2 增加 glogs.Debug 等直接方法 2020-05-06 14:34:31 +08:00
liangzy
8c78c2a6e2 v1.3.2 增加 glogs.Debug 等直接方法 2020-05-06 14:33:34 +08:00
liangzy
3a09faf4b8 v1.3.2 增加 glogs.Debug 等直接方法 2020-05-06 14:33:04 +08:00
liangzy
8998a4938d v1.3.2 增加 glogs.Debug 等直接方法 2020-05-06 14:32:26 +08:00
10 changed files with 346 additions and 116 deletions

View File

@ -23,7 +23,7 @@ logger.Critical("出错了")
logger.Info("出错了")
```
4. 如果需要写入es 必须设置环境变量 `GRLOG_APP_NAME`, 不能有反斜杠, 如
4. 如果需要写入es 或 alils 必须设置环境变量 `GRLOG_APP_NAME`, 不能有反斜杠, 如
```shell script
export GRLOG_APP_NAME=mkt.gaore.com;
```
@ -32,10 +32,20 @@ export GRLOG_APP_NAME=mkt.gaore.com;
import _ "golib.gaore.com/GaoreGo/grlogs/logs/es"
```
```go
import _ "golib.gaore.com/GaoreGo/grlogs/logs/alils"
```
5. 文件日志会写入到 `./runtime/logs/` 文件夹 **请务必在项目构建阶段创建该目录**
6. 完整示例
6. `AliLS` 日志接入的是阿里SLS GOSDK , 下面是关于 [https://github.com/aliyun/aliyun-log-go-sdk/tree/master/producer] 描述
producer提供了两种关闭模式分为有限关闭和安全关闭安全关闭会等待producer中缓存的所有的数据全部发送完成以后在关闭producer有限关闭会接收用户传递的一个参数值时间单位为秒当开始关闭producer的时候开始计时超过传递的设定值还未能完全关闭producer的话会强制退出producer此时可能会有部分数据未被成功发送而丢失。
所以用了aliLS 的 adapter 时,最好调用`grlogs.Close()` 或 `grlogs.CloseAll()` 方法安全关闭通道,以刷新缓冲区
7. 完整示例
```go
package grlogs
@ -48,30 +58,35 @@ import (
func TestGetLogger(t *testing.T) {
// 新建 channel 大小为128 标识为nds 日志通道 , Get 的方法 默认带 console 和 file 输出
l := Get("nds", 128).SetAdapter(LevelAll, AdapterElasticSearch)
l := grlogs.Get("nds", 128).SetAdapter(LevelAll, AdapterElasticSearch)
l.Debug("我正在调试")
l.Critical("出错了")
// 复用 nds 的日志通道
Get("nds").Warning("hadoee %s", time.Now().Format(time.RFC1123))
Get("nds").Warning("hadoee %s", time.Now().Format(time.RFC1123))
grlogs.Get("nds").Warning("hadoee %s", time.Now().Format(time.RFC1123))
grlogs.Get("nds").Warning("hadoee %s", time.Now().Format(time.RFC1123))
// 新建 channel 大小为默认 标识为wifi 日志通道 , GetEs 的方法 默认带 console 和 file 和 elatisearch 输出
GetEs("wifi")
grlogs.GetEs("wifi")
for i := 0; i < 10; i++ {
Get("wifi").Warning("Warning")
Get("wifi").Warn("Warn")
Get("wifi").Debug("Debug")
Get("wifi").Error("Error")
Get("wifi").Notice("Notice")
Get("wifi").Info("Info")
Get("wifi").Alert("Alert")
grlogs.Get("wifi").Warning("Warning")
grlogs.Get("wifi").Warn("Warn")
grlogs.Get("wifi").Debug("Debug")
grlogs.Get("wifi").Error("Error")
grlogs.Get("wifi").Notice("Notice")
grlogs.Get("wifi").Info("Info")
grlogs.Get("wifi").Alert("Alert")
}
Get("wifi").Critical("neoweiwoewe")
}
func TestGetEs(t *testing.T) {
fmt.Println("hello world")
func TestDropAdapter(t *testing.T) {
grlogs.SetAdapter(LevelAll, AdapterAliLs)
grlogs.DropAdapter(AdapterAliLs)
grlogs.Informational(errors.New("he hello"))
grlogs.SetAdapter(LevelAll, AdapterAliLs)
grlogs.Debug(errors.New("he hello"))
grlogs.CloseAll()
}
```

View File

@ -5,9 +5,21 @@ import (
"github.com/astaxie/beego/logs"
"os"
"path"
"strings"
)
var adatperMapper = map[Adapter]func(l *Logger, level Level) error{
var adatperSetMapper = map[Adapter]func(l *Logger, level Level) error{
AdapterSocket: func(l *Logger, level Level) error {
c := ConnLogConfig{
ReconnectOnMsg: false,
Reconnect: true,
Net: "",
Addr: "127.0.0.1:9888",
Level: level,
}
return l.SetLogger(logs.AdapterConn, c.String())
},
AdapterFile: func(l *Logger, level Level) error {
if wd, err := os.Getwd(); err == nil {
@ -33,46 +45,95 @@ var adatperMapper = map[Adapter]func(l *Logger, level Level) error{
},
AdapterElasticSearch: func(l *Logger, level Level) error {
dsn := "http://es-cn-0pp1mm3hq000dnbh4.public.elasticsearch.aliyuncs.com:9200/"
if os.Getenv(envkey) == "prod" || os.Getenv(envkey) == "" || os.Getenv(envkey) == "gray" {
dsn = "http://es-cn-0pp1mm3hq000dnbh4.elasticsearch.aliyuncs.com:9200/"
}
c := EsConfig{
Username: "elastic",
Password: "Hellogaore@",
Dsn: dsn,
Level: level,
Index: os.Getenv("GRLOG_APP_NAME"),
}
c := GenEsConfig(level)
return l.SetLogger(logs.AdapterEs, c.String())
},
AdapterAliLs: func(l *Logger, level Level) error {
var project string = "gaore-app-logstore"
var endpoint string
if os.Getenv(envkey) == "prod" || os.Getenv(envkey) == "" || os.Getenv(envkey) == "gray" {
endpoint = project + ".cn-shenzhen-intranet.log.aliyuncs.com"
} else if os.Getenv(envkey) == "dev" {
endpoint = project + ".cn-shenzhen.log.aliyuncs.com"
}
c := AliLSConfig{
Project: project,
Endpoint: endpoint,
KeyID: "LTAI4GCHwcqtrFD4DHRHxR4k",
KeySecret: "Ln19xfVYy6OMlJeF9aBvFl4fhRUKBl",
LogStore: "gaore-app-logstore",
Topics: []string{os.Getenv("GRLOG_APP_NAME")},
Source: "",
Level: level,
FlushWhen: 0,
}
c := GenAliConfig(level)
return l.SetLogger(logs.AdapterAliLS, c.String())
},
}
func GenAliConfig(level Level) *AliLSConfig {
var project string = "gaore-app-logstore"
var endpoint string
if os.Getenv(envkey) == "prod" || os.Getenv(envkey) == "" || os.Getenv(envkey) == "gray" {
endpoint = project + ".cn-shenzhen-intranet.log.aliyuncs.com"
} else if os.Getenv(envkey) == "dev" {
endpoint = project + ".cn-shenzhen.log.aliyuncs.com"
}
c := &AliLSConfig{
Project: project,
Endpoint: endpoint,
KeyID: "LTAI4GCHwcqtrFD4DHRHxR4k",
KeySecret: "Ln19xfVYy6OMlJeF9aBvFl4fhRUKBl",
LogStore: "gaore-app-logstore",
Topics: []string{os.Getenv("GRLOG_APP_NAME")},
Source: "",
Level: level,
FlushWhen: 0,
}
return c
}
func GenEsConfig(level Level) *EsConfig {
dsn := "http://es-cn-tl32xlfmu00015h34.public.elasticsearch.aliyuncs.com:9200/"
if os.Getenv(envkey) == "prod" || os.Getenv(envkey) == "" || os.Getenv(envkey) == "gray" {
dsn = "http://es-cn-tl32xlfmu00015h34.elasticsearch.aliyuncs.com:9200/"
}
c := &EsConfig{
Username: "kaifa_api",
Password: "2quYX3bTeahO",
Dsn: dsn,
Level: level,
Index: os.Getenv("GRLOG_APP_NAME"),
}
return c
}
var adatperDropMapper = map[Adapter]func(l *Logger) error{
AdapterAliLs: func(l *Logger) error {
return l.BeeLogger.DelLogger(logs.AdapterAliLS)
},
AdapterFile: func(l *Logger) error {
return l.BeeLogger.DelLogger(logs.AdapterFile)
},
AdapterConsole: func(l *Logger) error {
return l.BeeLogger.DelLogger(logs.AdapterConsole)
},
AdapterElasticSearch: func(l *Logger) error {
return l.BeeLogger.DelLogger(logs.AdapterEs)
},
}
func formatLog(f interface{}, v ...interface{}) string {
var msg string
switch f.(type) {
case string:
msg = f.(string)
if len(v) == 0 {
return msg
}
if strings.Contains(msg, "%") && !strings.Contains(msg, "%%") {
//format string
} else {
//do not contain format char
msg += strings.Repeat(" %v", len(v))
}
default:
msg = fmt.Sprint(f)
if len(v) == 0 {
return msg
}
msg += strings.Repeat(" %v", len(v))
}
return fmt.Sprintf(msg, v...)
}

59
grlogs.go Normal file
View File

@ -0,0 +1,59 @@
package grlogs
import "strings"
func Info(v ...interface{}) {
Get("grlogs").Info(generateFmtStr(len(v)), v...)
}
func Informational(v ...interface{}) {
Get("grlogs").Informational(generateFmtStr(len(v)), v...)
}
func Warning(v ...interface{}) {
Get("grlogs").Warning(generateFmtStr(len(v)), v...)
}
func Warn(v ...interface{}) {
Get("grlogs").Warn(generateFmtStr(len(v)), v...)
}
func Notice(v ...interface{}) {
Get("grlogs").Notice(generateFmtStr(len(v)), v...)
}
func Error(v ...interface{}) {
Get("grlogs").Error(generateFmtStr(len(v)), v...)
}
func Critical(v ...interface{}) {
Get("grlogs").Critical(generateFmtStr(len(v)), v...)
}
func Alert(v ...interface{}) {
Get("grlogs").Alert(generateFmtStr(len(v)), v...)
}
func Emergency(v ...interface{}) {
Get("grlogs").Emergency(generateFmtStr(len(v)), v...)
}
func Trace(v ...interface{}) {
Get("grlogs").Trace(generateFmtStr(len(v)), v...)
}
func Debug(v ...interface{}) {
Get("grlogs").Debug(generateFmtStr(len(v)), v...)
}
func generateFmtStr(n int) string {
return strings.Repeat("%v ", n)
}
func SetAdapter(level Level, adapter Adapter) {
Get("grlogs").SetAdapter(level, adapter)
}
func DropAdapter(adapter Adapter) {
Get("grlogs").DropAdapter(adapter)
}

122
log.go
View File

@ -11,13 +11,21 @@ var loggers = sync.Map{}
var envkey = "CENTER_RUNMODE"
var defaultModeMapping = map[string][]AdapterTupple{
"es": []AdapterTupple{{LevelAll, AdapterConsole}, {LevelAll, AdapterElasticSearch}},
"ali": []AdapterTupple{{LevelAll, AdapterConsole}, {LevelAll, AdapterAliLs}},
"": []AdapterTupple{{LevelAll, AdapterConsole}, {LevelAll, AdapterFile}},
}
type Logger struct {
Lable string
*logs.BeeLogger
}
type Getter func() *Logger
func (self *Logger) SetAdapter(level Level, adapter Adapter) *Logger {
if call, ok := adatperMapper[adapter]; ok {
if call, ok := adatperSetMapper[adapter]; ok {
if err := call(self, level); err != nil {
fmt.Println(err)
}
@ -25,65 +33,70 @@ func (self *Logger) SetAdapter(level Level, adapter Adapter) *Logger {
return self
}
func New(label string, channelLens ...int64) (l *Logger, loaded bool) {
var channellens int64
var tmp interface{}
tmp, loaded = loggers.LoadOrStore(label, new(Logger))
l = tmp.(*Logger)
if len(channelLens) > 0 {
channellens = channelLens[0]
func (self *Logger) DropAdapter(adapter Adapter) *Logger {
if call, ok := adatperDropMapper[adapter]; ok {
if err := call(self); err != nil {
fmt.Println(err)
}
}
if !loaded {
l.Lable = label
l.BeeLogger = logs.NewLogger(channellens)
l.BeeLogger.Lable = label
l.Env = os.Getenv(envkey)
l.SetPrefix(fmt.Sprintf("[env:%s logger:%s]", os.Getenv(envkey), label))
l.EnableFuncCallDepth(true)
l.SetLogFuncCallDepth(2)
}
return
return self
}
func Get(label string, channelLens ...int64) (l *Logger) {
var filelevel Level = LevelInfo
var loaded bool
if os.Getenv(envkey) == "dev" {
filelevel = LevelAll
func newLoggerFromMap(label string, defaultmode string, channelLens ...int64) Getter {
if tmp, ok := loggers.Load(label); ok {
return tmp.(Getter)
}
if l, loaded = New(label, channelLens...); !loaded {
l.SetAdapter(filelevel, AdapterFile).SetAdapter(LevelAll, AdapterConsole)
var l *Logger
var once sync.Once
wapperGetter := Getter(func() *Logger {
once.Do(func() {
var channelLensNum int64 = 100
if len(channelLens) > 0 && channelLens[0] > 0 {
channelLensNum = channelLens[0]
}
l = &Logger{BeeLogger: logs.NewLogger(channelLensNum)}
l.Lable = label
l.BeeLogger = logs.NewLogger(channelLensNum)
l.BeeLogger.Lable = label
l.Env = os.Getenv(envkey)
l.SetPrefix(fmt.Sprintf("[env:%s logger:%s]", os.Getenv(envkey), label))
l.EnableFuncCallDepth(true)
l.SetLogFuncCallDepth(2)
if mode, ok := defaultModeMapping[defaultmode]; ok {
for _, v := range mode {
l.SetAdapter(v.Level, v.Adapter)
}
}
})
return l
})
tmp, loaded := loggers.LoadOrStore(label, wapperGetter)
if loaded {
return tmp.(Getter)
}
return
return wapperGetter
}
func GetEs(label string, channelLens ...int64) (l *Logger) {
var filelevel Level = LevelInfo
var loaded bool
if os.Getenv(envkey) == "dev" {
filelevel = LevelAll
}
if l, loaded = New(label, channelLens...); !loaded {
l.SetAdapter(filelevel, AdapterFile).SetAdapter(LevelAll, AdapterConsole).SetAdapter(filelevel, AdapterElasticSearch)
}
return
func Get(label string, channelLens ...int64) *Logger {
getter := newLoggerFromMap(label, "", channelLens...)
return getter()
}
func GetAli(label string, channelLens ...int64) (l *Logger) {
var filelevel Level = LevelInfo
var loaded bool
if os.Getenv(envkey) == "dev" {
filelevel = LevelAll
}
func GetEs(label string, channelLens ...int64) *Logger {
getter := newLoggerFromMap(label, "es", channelLens...)
return getter()
}
if l, loaded = New(label, channelLens...); !loaded {
l.SetAdapter(LevelAll, AdapterConsole).SetAdapter(filelevel, AdapterFile).SetAdapter(filelevel, AdapterAliLs)
}
return
func GetAli(label string, channelLens ...int64) *Logger {
getter := newLoggerFromMap(label, "ali", channelLens...)
return getter()
}
func Close(lables ...string) {
@ -119,3 +132,14 @@ func CloseAll() {
})
wg.Wait()
}
func init() {
var level Level = LevelInfo
if os.Getenv(envkey) == "dev" {
level = LevelAll
}
defaultModeMapping["es"] = []AdapterTupple{{level, AdapterConsole}, {level, AdapterElasticSearch}}
defaultModeMapping["ali"] = []AdapterTupple{{level, AdapterConsole}, {level, AdapterAliLs}}
defaultModeMapping[""] = []AdapterTupple{{level, AdapterConsole}, {level, AdapterFile}}
}

View File

@ -1,8 +1,10 @@
package grlogs
import (
"errors"
_ "golib.gaore.com/GaoreGo/grlogs/logs/alils"
_ "golib.gaore.com/GaoreGo/grlogs/logs/es"
"sync"
"testing"
"time"
)
@ -47,5 +49,50 @@ func TestGetAliLs(t *testing.T) {
l.Warn("Warn")
}
Close(lable, lable, lable)
time.Sleep(time.Hour * 1)
}
func TestDropAdapter(t *testing.T) {
SetAdapter(LevelAll, AdapterAliLs)
DropAdapter(AdapterAliLs)
Informational(errors.New("he hello"))
SetAdapter(LevelAll, AdapterAliLs)
Debug(errors.New("he hello"))
CloseAll()
}
func TestNew(t *testing.T) {
wg := sync.WaitGroup{}
for i := 0; i < 10; i++ {
wg.Add(1)
go func(i int) {
GetAli("ok").Debug("%d", i)
time.Sleep(time.Second * 10)
wg.Done()
}(i)
}
for i := 0; i < 10; i++ {
GetAli("ok").Debug("aaaaaa%d", i)
}
wg.Wait()
}
func TestGetEs(t *testing.T) {
wg := sync.WaitGroup{}
for i := 0; i < 10; i++ {
wg.Add(1)
go func(i int) {
GetEs("ok").Debug("%d", i)
time.Sleep(time.Second * 10)
wg.Done()
}(i)
}
for i := 0; i < 10; i++ {
GetEs("ok").Debug("aaaaaa%d", i)
}
wg.Wait()
}

View File

@ -5,6 +5,8 @@ import (
"fmt"
"github.com/aliyun/aliyun-log-go-sdk/producer"
"golib.gaore.com/GaoreGo/grlogs/logs"
"os"
"runtime"
"time"
)
@ -27,6 +29,7 @@ type alilsLogger struct {
Source string `json:"source"`
Level int `json:"level"`
FlushWhen int `json:"flush_when"`
Debug bool
}
func (a *alilsLogger) Init(jsonconfig string) error {
@ -40,11 +43,14 @@ func (a *alilsLogger) Init(jsonconfig string) error {
producerConfig.AccessKeyID = a.KeyID
producerConfig.AccessKeySecret = a.KeySecret
producerConfig.LingerMs = 100
producerConfig.NoRetryStatusCodeList = []int{-1}
producerConfig.Retries = 2
producerConfig.AllowLogLevel = "error"
producerConfig.MaxIoWorkerCount = int64(runtime.NumCPU())
a.producer = producer.InitProducer(producerConfig)
a.callback = &Callback{}
a.producer.Start()
a.Debug = os.Getenv("GRLOG_ALILS_DEBUG") == "on"
return nil
}
@ -62,12 +68,19 @@ func (a *alilsLogger) WriteMsg(when time.Time, msg string, level int, lable stri
log := producer.GenerateLog(uint32(when.Unix()), vals)
for _, topic := range a.Topics {
if err := a.producer.SendLog(a.Project, a.LogStore, topic, a.Source, log); err != nil {
return err
if a.Debug {
for _, topic := range a.Topics {
if err := a.producer.SendLogWithCallBack(a.Project, a.LogStore, topic, a.Source, log, a.callback); err != nil {
return err
}
}
} else {
for _, topic := range a.Topics {
if err := a.producer.SendLog(a.Project, a.LogStore, topic, a.Source, log); err != nil {
return err
}
}
}
return nil
}

View File

@ -11,15 +11,15 @@ type Callback struct {
func (callback *Callback) Success(result *producer.Result) {
attemptList := result.GetReservedAttempts()
for _, attempt := range attemptList {
fmt.Printf("%+v \n", attempt)
fmt.Printf("alilog %+v \n", attempt)
}
}
func (callback *Callback) Fail(result *producer.Result) {
fmt.Println(result.IsSuccessful())
fmt.Println(result.GetErrorCode())
fmt.Println(result.GetErrorMessage())
fmt.Println(result.GetReservedAttempts())
fmt.Println(result.GetRequestId())
fmt.Println(result.GetTimeStampMs())
fmt.Println("IsSuccessful", result.IsSuccessful())
fmt.Println("GetErrorCode", result.GetErrorCode())
fmt.Println("GetErrorMessage", result.GetErrorMessage())
fmt.Println("GetReservedAttempts", result.GetReservedAttempts())
fmt.Println("GetRequestId", result.GetRequestId())
fmt.Println("GetTimeStampMs", result.GetTimeStampMs())
}

View File

@ -89,7 +89,7 @@ func (el *esLogger) WriteMsg(when time.Time, msg string, level int, lable string
d := goes.Document{
Index: fmt.Sprintf("%s-%04d.%02d.%02d", el.IndexName, when.Year(), when.Month(), when.Day()),
Type: "logs",
Type: "_doc",
Fields: vals,
}
_, err := el.Index(d, nil)

View File

@ -281,7 +281,7 @@ func testFileRotate(t *testing.T, fn1, fn2 string, daily, hourly bool) {
fw.hourlyOpenDate = fw.hourlyOpenTime.Day()
}
fw.WriteMsg(time.Now(), "this is a msg for test", LevelDebug)
fw.WriteMsg(time.Now(), "this is a msg for test", LevelDebug, "", "dev")
for _, file := range []string{fn1, fn2} {
_, err := os.Stat(file)

View File

@ -4,6 +4,11 @@ type Level int
type Adapter string
type AdapterTupple struct {
Level Level
Adapter Adapter
}
const (
_ Level = iota
LevelNone Level = iota
@ -23,3 +28,9 @@ const (
AdapterElasticSearch Adapter = "es"
AdapterAliLs Adapter = "alils"
)
const (
DEV = "dev"
GRAY = "gray"
PROD = "prod"
)