diff --git a/config.go b/config.go index 12d02f1..2ecdc42 100644 --- a/config.go +++ b/config.go @@ -54,3 +54,20 @@ func (c *EsConfig) String() string { b, _ := json.Marshal(c) return bytes.NewBuffer(b).String() } + +type AliLSConfig struct { + Project string `json:"project"` + Endpoint string `json:"endpoint"` + KeyID string `json:"key_id"` + KeySecret string `json:"key_secret"` + LogStore string `json:"log_store"` + Topics []string `json:"topics"` + Source string `json:"source"` + Level Level `json:"level"` + FlushWhen int `json:"flush_when"` +} + +func (c *AliLSConfig) String() string { + b, _ := json.Marshal(c) + return bytes.NewBuffer(b).String() +} diff --git a/funtion.go b/funtion.go index 9d42288..18eb125 100644 --- a/funtion.go +++ b/funtion.go @@ -49,4 +49,21 @@ var adatperMapper = map[Adapter]func(l *Logger, level Level) error{ return l.SetLogger(logs.AdapterEs, c.String()) }, + + AdapterAliLs: func(l *Logger, level Level) error { + + c := AliLSConfig{ + Project: "gaore-app-logstore", + Endpoint: "gaore-app-logstore.cn-shenzhen.log.aliyuncs.com", + KeyID: "LTAI4GCHwcqtrFD4DHRHxR4k", + KeySecret: "Ln19xfVYy6OMlJeF9aBvFl4fhRUKBl", + LogStore: "gaore-app-logstore", + Topics: nil, + Source: "", + Level: level, + FlushWhen: 0, + } + + return l.SetLogger(logs.AdapterAliLS, c.String()) + }, } diff --git a/log_test.go b/log_test.go index 3de85b4..1497f31 100644 --- a/log_test.go +++ b/log_test.go @@ -1,7 +1,7 @@ package grlogs import ( - "fmt" + _ "golib.gaore.com/GaoreGo/grlogs/logs/alils" _ "golib.gaore.com/GaoreGo/grlogs/logs/es" "testing" "time" @@ -32,6 +32,11 @@ func TestGetLogger(t *testing.T) { Get("wifi").Critical("neoweiwoewe") } -func TestGetEs(t *testing.T) { - fmt.Println("hello world") +func TestGetAliLs(t *testing.T) { + l := Get("es").SetAdapter(LevelAll, AdapterAliLs) + l.Info("endport") + l.Info("endport") + l.Info("endport") + l.Info("endport") + time.Sleep(time.Millisecond * 500) } diff --git a/logs/alils/alils.go b/logs/alils/alils.go index 24910a5..bcc7fa0 100644 --- a/logs/alils/alils.go +++ b/logs/alils/alils.go @@ -2,23 +2,22 @@ package alils import ( "encoding/json" - "strings" - "sync" - "time" - - "github.com/gogo/protobuf/proto" + "fmt" + "github.com/aliyun/aliyun-log-go-sdk/producer" "golib.gaore.com/GaoreGo/grlogs/logs" + "time" ) -const ( - // CacheSize set the flush size - CacheSize int = 64 - // Delimiter define the topic delimiter - Delimiter string = "##" -) +func NewAliLS() logs.Logger { + cw := &alilsLogger{ + Level: logs.LevelDebug, + } + return cw +} -// Config is the Config for Ali Log -type Config struct { +type alilsLogger struct { + producer *producer.Producer + callback *Callback Project string `json:"project"` Endpoint string `json:"endpoint"` KeyID string `json:"key_id"` @@ -30,155 +29,49 @@ type Config struct { FlushWhen int `json:"flush_when"` } -// aliLSWriter implements LoggerInterface. -// it writes messages in keep-live tcp connection. -type aliLSWriter struct { - store *LogStore - group []*LogGroup - withMap bool - groupMap map[string]*LogGroup - lock *sync.Mutex - Config -} - -// NewAliLS create a new Logger -func NewAliLS() logs.Logger { - alils := new(aliLSWriter) - alils.Level = logs.LevelTrace - return alils -} - -// Init parse config and init struct -func (c *aliLSWriter) Init(jsonConfig string) (err error) { +func (a *alilsLogger) Init(jsonconfig string) error { - json.Unmarshal([]byte(jsonConfig), c) - - if c.FlushWhen > CacheSize { - c.FlushWhen = CacheSize - } - - prj := &LogProject{ - Name: c.Project, - Endpoint: c.Endpoint, - AccessKeyID: c.KeyID, - AccessKeySecret: c.KeySecret, - } - - c.store, err = prj.GetLogStore(c.LogStore) + err := json.Unmarshal([]byte(jsonconfig), a) if err != nil { return err } - // Create default Log Group - c.group = append(c.group, &LogGroup{ - Topic: proto.String(""), - Source: proto.String(c.Source), - Logs: make([]*Log, 0, c.FlushWhen), - }) - - // Create other Log Group - c.groupMap = make(map[string]*LogGroup) - for _, topic := range c.Topics { - - lg := &LogGroup{ - Topic: proto.String(topic), - Source: proto.String(c.Source), - Logs: make([]*Log, 0, c.FlushWhen), - } - - c.group = append(c.group, lg) - c.groupMap[topic] = lg - } - - if len(c.group) == 1 { - c.withMap = false - } else { - c.withMap = true - } - - c.lock = &sync.Mutex{} + producerConfig := producer.GetDefaultProducerConfig() + producerConfig.Endpoint = a.Endpoint + producerConfig.AccessKeyID = a.KeyID + producerConfig.AccessKeySecret = a.KeySecret + producerConfig.LingerMs = 100 + a.producer = producer.InitProducer(producerConfig) + a.callback = &Callback{} + a.producer.Start() return nil } -// WriteMsg write message in connection. -// if connection is down, try to re-connect. -func (c *aliLSWriter) WriteMsg(when time.Time, msg string, level int, lable string, env string) (err error) { - - if level > c.Level { - return nil - } - - var topic string - var content string - var lg *LogGroup - if c.withMap { - - // Topic，LogGroup - strs := strings.SplitN(msg, Delimiter, 2) - if len(strs) == 2 { - pos := strings.LastIndex(strs[0], " ") - topic = strs[0][pos+1 : len(strs[0])] - content = strs[0][0:pos] + strs[1] - lg = c.groupMap[topic] - } - - // send to empty Topic - if lg == nil { - content = msg - lg = c.group[0] - } - } else { - content = msg - lg = c.group[0] - } - - c1 := &LogContent{ - Key: proto.String("msg"), - Value: proto.String(content), - } - - l := &Log{ - Time: proto.Uint32(uint32(when.Unix())), - Contents: []*LogContent{ - c1, - }, - } - - c.lock.Lock() - lg.Logs = append(lg.Logs, l) - c.lock.Unlock() - - if len(lg.Logs) >= c.FlushWhen { - c.flush(lg) - } - - return nil +func (a *alilsLogger) WriteMsg(when time.Time, msg string, level int, lable string, env string) error { + + vals := map[string]string{"msg": msg, "level": fmt.Sprintf("%d", level)} + vals["level_string"] = logs.GetLevelString(level) + vals["env"] = env + vals["lable"] = lable + vals["hostname"] = GetHostname() + vals["working_idr"] = Getwd() + vals["home_dir"] = GetUserHomename() + vals["hardware_addr"] = GetCurrentInterfaceHardwareAddr() + vals["client_addrs"] = GetCurrentInterfaceAddrs() + + log := producer.GenerateLog(uint32(when.Unix()), vals) + err := a.producer.SendLog(a.Project, a.LogStore, "topic", "127.0.0.1", log) + return err } -// Flush implementing method. empty. -func (c *aliLSWriter) Flush() { - - // flush all group - for _, lg := range c.group { - c.flush(lg) - } +func (a *alilsLogger) Destroy() { + a.producer.Close(60) + a.producer.SafeClose() } -// Destroy destroy connection writer and close tcp listener. -func (c *aliLSWriter) Destroy() { -} - -func (c *aliLSWriter) flush(lg *LogGroup) { - - c.lock.Lock() - defer c.lock.Unlock() - err := c.store.PutLogs(lg) - if err != nil { - return - } +func (a *alilsLogger) Flush() { - lg.Logs = make([]*Log, 0, c.FlushWhen) } func init() { diff --git a/logs/alils/callback.go b/logs/alils/callback.go new file mode 100644 index 0000000..9e20a54 --- /dev/null +++ b/logs/alils/callback.go @@ -0,0 +1,25 @@ +package alils + +import ( + "fmt" + "github.com/aliyun/aliyun-log-go-sdk/producer" +) + +type Callback struct { +} + +func (callback *Callback) Success(result *producer.Result) { + attemptList := result.GetReservedAttempts() + for _, attempt := range attemptList { + fmt.Printf("%+v \n", attempt) + } +} + +func (callback *Callback) Fail(result *producer.Result) { + fmt.Println(result.IsSuccessful()) + fmt.Println(result.GetErrorCode()) + fmt.Println(result.GetErrorMessage()) + fmt.Println(result.GetReservedAttempts()) + fmt.Println(result.GetRequestId()) + fmt.Println(result.GetTimeStampMs()) +} diff --git a/logs/alils/config.go b/logs/alils/config.go deleted file mode 100755 index e8c2444..0000000 --- a/logs/alils/config.go +++ /dev/null @@ -1,13 +0,0 @@ -package alils - -const ( - version = "0.5.0" // SDK version - signatureMethod = "hmac-sha1" // Signature method - - // OffsetNewest stands for the log head offset, i.e. the offset that will be - // assigned to the next message that will be produced to the shard. - OffsetNewest = "end" - // OffsetOldest stands for the oldest offset available on the logstore for a - // shard. - OffsetOldest = "begin" -) diff --git a/logs/alils/log.pb.go b/logs/alils/log.pb.go deleted file mode 100755 index 601b0d7..0000000 --- a/logs/alils/log.pb.go +++ /dev/null @@ -1,1038 +0,0 @@ -package alils - -import ( - "fmt" - "io" - "math" - - "github.com/gogo/protobuf/proto" - github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -var ( - // ErrInvalidLengthLog invalid proto - ErrInvalidLengthLog = fmt.Errorf("proto: negative length found during unmarshaling") - // ErrIntOverflowLog overflow - ErrIntOverflowLog = fmt.Errorf("proto: integer overflow") -) - -// Log define the proto Log -type Log struct { - Time *uint32 `protobuf:"varint,1,req,name=Time" json:"Time,omitempty"` - Contents []*LogContent `protobuf:"bytes,2,rep,name=Contents" json:"Contents,omitempty"` - XXXUnrecognized []byte `json:"-"` -} - -// Reset the Log -func (m *Log) Reset() { *m = Log{} } - -// String return the Compact Log -func (m *Log) String() string { return proto.CompactTextString(m) } - -// ProtoMessage not implemented -func (*Log) ProtoMessage() {} - -// GetTime return the Log's Time -func (m *Log) GetTime() uint32 { - if m != nil && m.Time != nil { - return *m.Time - } - return 0 -} - -// GetContents return the Log's Contents -func (m *Log) GetContents() []*LogContent { - if m != nil { - return m.Contents - } - return nil -} - -// LogContent define the Log content struct -type LogContent struct { - Key *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"` - Value *string `protobuf:"bytes,2,req,name=Value" json:"Value,omitempty"` - XXXUnrecognized []byte `json:"-"` -} - -// Reset LogContent -func (m *LogContent) Reset() { *m = LogContent{} } - -// String return the compact text -func (m *LogContent) String() string { return proto.CompactTextString(m) } - -// ProtoMessage not implemented -func (*LogContent) ProtoMessage() {} - -// GetKey return the Key -func (m *LogContent) GetKey() string { - if m != nil && m.Key != nil { - return *m.Key - } - return "" -} - -// GetValue return the Value -func (m *LogContent) GetValue() string { - if m != nil && m.Value != nil { - return *m.Value - } - return "" -} - -// LogGroup define the logs struct -type LogGroup struct { - Logs []*Log `protobuf:"bytes,1,rep,name=Logs" json:"Logs,omitempty"` - Reserved *string `protobuf:"bytes,2,opt,name=Reserved" json:"Reserved,omitempty"` - Topic *string `protobuf:"bytes,3,opt,name=Topic" json:"Topic,omitempty"` - Source *string `protobuf:"bytes,4,opt,name=Source" json:"Source,omitempty"` - XXXUnrecognized []byte `json:"-"` -} - -// Reset LogGroup -func (m *LogGroup) Reset() { *m = LogGroup{} } - -// String return the compact text -func (m *LogGroup) String() string { return proto.CompactTextString(m) } - -// ProtoMessage not implemented -func (*LogGroup) ProtoMessage() {} - -// GetLogs return the loggroup logs -func (m *LogGroup) GetLogs() []*Log { - if m != nil { - return m.Logs - } - return nil -} - -// GetReserved return Reserved -func (m *LogGroup) GetReserved() string { - if m != nil && m.Reserved != nil { - return *m.Reserved - } - return "" -} - -// GetTopic return Topic -func (m *LogGroup) GetTopic() string { - if m != nil && m.Topic != nil { - return *m.Topic - } - return "" -} - -// GetSource return Source -func (m *LogGroup) GetSource() string { - if m != nil && m.Source != nil { - return *m.Source - } - return "" -} - -// LogGroupList define the LogGroups -type LogGroupList struct { - LogGroups []*LogGroup `protobuf:"bytes,1,rep,name=logGroups" json:"logGroups,omitempty"` - XXXUnrecognized []byte `json:"-"` -} - -// Reset LogGroupList -func (m *LogGroupList) Reset() { *m = LogGroupList{} } - -// String return compact text -func (m *LogGroupList) String() string { return proto.CompactTextString(m) } - -// ProtoMessage not implemented -func (*LogGroupList) ProtoMessage() {} - -// GetLogGroups return the LogGroups -func (m *LogGroupList) GetLogGroups() []*LogGroup { - if m != nil { - return m.LogGroups - } - return nil -} - -// Marshal the logs to byte slice -func (m *Log) Marshal() (data []byte, err error) { - size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) - if err != nil { - return nil, err - } - return data[:n], nil -} - -// MarshalTo data -func (m *Log) MarshalTo(data []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Time == nil { - return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Time") - } - data[i] = 0x8 - i++ - i = encodeVarintLog(data, i, uint64(*m.Time)) - if len(m.Contents) > 0 { - for _, msg := range m.Contents { - data[i] = 0x12 - i++ - i = encodeVarintLog(data, i, uint64(msg.Size())) - n, err := msg.MarshalTo(data[i:]) - if err != nil { - return 0, err - } - i += n - } - } - if m.XXXUnrecognized != nil { - i += copy(data[i:], m.XXXUnrecognized) - } - return i, nil -} - -// Marshal LogContent -func (m *LogContent) Marshal() (data []byte, err error) { - size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) - if err != nil { - return nil, err - } - return data[:n], nil -} - -// MarshalTo logcontent to data -func (m *LogContent) MarshalTo(data []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Key == nil { - return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Key") - } - data[i] = 0xa - i++ - i = encodeVarintLog(data, i, uint64(len(*m.Key))) - i += copy(data[i:], *m.Key) - - if m.Value == nil { - return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("Value") - } - data[i] = 0x12 - i++ - i = encodeVarintLog(data, i, uint64(len(*m.Value))) - i += copy(data[i:], *m.Value) - if m.XXXUnrecognized != nil { - i += copy(data[i:], m.XXXUnrecognized) - } - return i, nil -} - -// Marshal LogGroup -func (m *LogGroup) Marshal() (data []byte, err error) { - size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) - if err != nil { - return nil, err - } - return data[:n], nil -} - -// MarshalTo LogGroup to data -func (m *LogGroup) MarshalTo(data []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.Logs) > 0 { - for _, msg := range m.Logs { - data[i] = 0xa - i++ - i = encodeVarintLog(data, i, uint64(msg.Size())) - n, err := msg.MarshalTo(data[i:]) - if err != nil { - return 0, err - } - i += n - } - } - if m.Reserved != nil { - data[i] = 0x12 - i++ - i = encodeVarintLog(data, i, uint64(len(*m.Reserved))) - i += copy(data[i:], *m.Reserved) - } - if m.Topic != nil { - data[i] = 0x1a - i++ - i = encodeVarintLog(data, i, uint64(len(*m.Topic))) - i += copy(data[i:], *m.Topic) - } - if m.Source != nil { - data[i] = 0x22 - i++ - i = encodeVarintLog(data, i, uint64(len(*m.Source))) - i += copy(data[i:], *m.Source) - } - if m.XXXUnrecognized != nil { - i += copy(data[i:], m.XXXUnrecognized) - } - return i, nil -} - -// Marshal LogGroupList -func (m *LogGroupList) Marshal() (data []byte, err error) { - size := m.Size() - data = make([]byte, size) - n, err := m.MarshalTo(data) - if err != nil { - return nil, err - } - return data[:n], nil -} - -// MarshalTo LogGroupList to data -func (m *LogGroupList) MarshalTo(data []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if len(m.LogGroups) > 0 { - for _, msg := range m.LogGroups { - data[i] = 0xa - i++ - i = encodeVarintLog(data, i, uint64(msg.Size())) - n, err := msg.MarshalTo(data[i:]) - if err != nil { - return 0, err - } - i += n - } - } - if m.XXXUnrecognized != nil { - i += copy(data[i:], m.XXXUnrecognized) - } - return i, nil -} - -func encodeFixed64Log(data []byte, offset int, v uint64) int { - data[offset] = uint8(v) - data[offset+1] = uint8(v >> 8) - data[offset+2] = uint8(v >> 16) - data[offset+3] = uint8(v >> 24) - data[offset+4] = uint8(v >> 32) - data[offset+5] = uint8(v >> 40) - data[offset+6] = uint8(v >> 48) - data[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Log(data []byte, offset int, v uint32) int { - data[offset] = uint8(v) - data[offset+1] = uint8(v >> 8) - data[offset+2] = uint8(v >> 16) - data[offset+3] = uint8(v >> 24) - return offset + 4 -} -func encodeVarintLog(data []byte, offset int, v uint64) int { - for v >= 1<<7 { - data[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - data[offset] = uint8(v) - return offset + 1 -} - -// Size return the log's size -func (m *Log) Size() (n int) { - var l int - _ = l - if m.Time != nil { - n += 1 + sovLog(uint64(*m.Time)) - } - if len(m.Contents) > 0 { - for _, e := range m.Contents { - l = e.Size() - n += 1 + l + sovLog(uint64(l)) - } - } - if m.XXXUnrecognized != nil { - n += len(m.XXXUnrecognized) - } - return n -} - -// Size return LogContent size based on Key and Value -func (m *LogContent) Size() (n int) { - var l int - _ = l - if m.Key != nil { - l = len(*m.Key) - n += 1 + l + sovLog(uint64(l)) - } - if m.Value != nil { - l = len(*m.Value) - n += 1 + l + sovLog(uint64(l)) - } - if m.XXXUnrecognized != nil { - n += len(m.XXXUnrecognized) - } - return n -} - -// Size return LogGroup size based on Logs -func (m *LogGroup) Size() (n int) { - var l int - _ = l - if len(m.Logs) > 0 { - for _, e := range m.Logs { - l = e.Size() - n += 1 + l + sovLog(uint64(l)) - } - } - if m.Reserved != nil { - l = len(*m.Reserved) - n += 1 + l + sovLog(uint64(l)) - } - if m.Topic != nil { - l = len(*m.Topic) - n += 1 + l + sovLog(uint64(l)) - } - if m.Source != nil { - l = len(*m.Source) - n += 1 + l + sovLog(uint64(l)) - } - if m.XXXUnrecognized != nil { - n += len(m.XXXUnrecognized) - } - return n -} - -// Size return LogGroupList size -func (m *LogGroupList) Size() (n int) { - var l int - _ = l - if len(m.LogGroups) > 0 { - for _, e := range m.LogGroups { - l = e.Size() - n += 1 + l + sovLog(uint64(l)) - } - } - if m.XXXUnrecognized != nil { - n += len(m.XXXUnrecognized) - } - return n -} - -func sovLog(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n -} -func sozLog(x uint64) (n int) { - return sovLog((x << 1) ^ (x >> 63)) -} - -// Unmarshal data to log -func (m *Log) Unmarshal(data []byte) error { - var hasFields [1]uint64 - l := len(data) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Log: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Log: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType) - } - var v uint32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - v |= (uint32(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.Time = &v - hasFields[0] |= uint64(0x00000001) - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Contents", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthLog - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Contents = append(m.Contents, &LogContent{}) - if err := m.Contents[len(m.Contents)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipLog(data[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthLog - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXXUnrecognized = append(m.XXXUnrecognized, data[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - if hasFields[0]&uint64(0x00000001) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Time") - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} - -// Unmarshal data to LogContent -func (m *LogContent) Unmarshal(data []byte) error { - var hasFields [1]uint64 - l := len(data) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Content: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Content: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthLog - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - s := string(data[iNdEx:postIndex]) - m.Key = &s - iNdEx = postIndex - hasFields[0] |= uint64(0x00000001) - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthLog - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - s := string(data[iNdEx:postIndex]) - m.Value = &s - iNdEx = postIndex - hasFields[0] |= uint64(0x00000002) - default: - iNdEx = preIndex - skippy, err := skipLog(data[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthLog - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXXUnrecognized = append(m.XXXUnrecognized, data[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - if hasFields[0]&uint64(0x00000001) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Key") - } - if hasFields[0]&uint64(0x00000002) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Value") - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} - -// Unmarshal data to LogGroup -func (m *LogGroup) Unmarshal(data []byte) error { - l := len(data) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: LogGroup: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: LogGroup: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Logs", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthLog - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Logs = append(m.Logs, &Log{}) - if err := m.Logs[len(m.Logs)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Reserved", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthLog - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - s := string(data[iNdEx:postIndex]) - m.Reserved = &s - iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthLog - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - s := string(data[iNdEx:postIndex]) - m.Topic = &s - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Source", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthLog - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - s := string(data[iNdEx:postIndex]) - m.Source = &s - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipLog(data[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthLog - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXXUnrecognized = append(m.XXXUnrecognized, data[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} - -// Unmarshal data to LogGroupList -func (m *LogGroupList) Unmarshal(data []byte) error { - l := len(data) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: LogGroupList: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: LogGroupList: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field LogGroups", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLog - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthLog - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.LogGroups = append(m.LogGroups, &LogGroup{}) - if err := m.LogGroups[len(m.LogGroups)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipLog(data[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthLog - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXXUnrecognized = append(m.XXXUnrecognized, data[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} - -func skipLog(data []byte) (n int, err error) { - l := len(data) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowLog - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowLog - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if data[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowLog - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - iNdEx += length - if length < 0 { - return 0, ErrInvalidLengthLog - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowLog - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := data[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipLog(data[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} diff --git a/logs/alils/log_config.go b/logs/alils/log_config.go deleted file mode 100755 index e8564ef..0000000 --- a/logs/alils/log_config.go +++ /dev/null @@ -1,42 +0,0 @@ -package alils - -// InputDetail define log detail -type InputDetail struct { - LogType string `json:"logType"` - LogPath string `json:"logPath"` - FilePattern string `json:"filePattern"` - LocalStorage bool `json:"localStorage"` - TimeFormat string `json:"timeFormat"` - LogBeginRegex string `json:"logBeginRegex"` - Regex string `json:"regex"` - Keys []string `json:"key"` - FilterKeys []string `json:"filterKey"` - FilterRegex []string `json:"filterRegex"` - TopicFormat string `json:"topicFormat"` -} - -// OutputDetail define the output detail -type OutputDetail struct { - Endpoint string `json:"endpoint"` - LogStoreName string `json:"logstoreName"` -} - -// LogConfig define Log Config -type LogConfig struct { - Name string `json:"configName"` - InputType string `json:"inputType"` - InputDetail InputDetail `json:"inputDetail"` - OutputType string `json:"outputType"` - OutputDetail OutputDetail `json:"outputDetail"` - - CreateTime uint32 - LastModifyTime uint32 - - project *LogProject -} - -// GetAppliedMachineGroup returns applied machine group of this config. -func (c *LogConfig) GetAppliedMachineGroup(confName string) (groupNames []string, err error) { - groupNames, err = c.project.GetAppliedMachineGroups(c.Name) - return -} diff --git a/logs/alils/log_project.go b/logs/alils/log_project.go deleted file mode 100755 index 59db8cb..0000000 --- a/logs/alils/log_project.go +++ /dev/null @@ -1,819 +0,0 @@ -/* -Package alils implements the SDK(v0.5.0) of Simple Log Service(abbr. SLS). - -For more description about SLS, please read this article: -http://gitlab.alibaba-inc.com/sls/doc. -*/ -package alils - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httputil" -) - -// Error message in SLS HTTP response. -type errorMessage struct { - Code string `json:"errorCode"` - Message string `json:"errorMessage"` -} - -// LogProject Define the Ali Project detail -type LogProject struct { - Name string // Project name - Endpoint string // IP or hostname of SLS endpoint - AccessKeyID string - AccessKeySecret string -} - -// NewLogProject creates a new SLS project. -func NewLogProject(name, endpoint, AccessKeyID, accessKeySecret string) (p *LogProject, err error) { - p = &LogProject{ - Name: name, - Endpoint: endpoint, - AccessKeyID: AccessKeyID, - AccessKeySecret: accessKeySecret, - } - return p, nil -} - -// ListLogStore returns all logstore names of project p. -func (p *LogProject) ListLogStore() (storeNames []string, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - uri := fmt.Sprintf("/logstores") - r, err := request(p, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to list logstore") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - type Body struct { - Count int - LogStores []string - } - body := &Body{} - - err = json.Unmarshal(buf, body) - if err != nil { - return - } - - storeNames = body.LogStores - - return -} - -// GetLogStore returns logstore according by logstore name. -func (p *LogProject) GetLogStore(name string) (s *LogStore, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - r, err := request(p, "GET", "/logstores/"+name, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to get logstore") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - s = &LogStore{} - err = json.Unmarshal(buf, s) - if err != nil { - return - } - s.project = p - return -} - -// CreateLogStore creates a new logstore in SLS, -// where name is logstore name, -// and ttl is time-to-live(in day) of logs, -// and shardCnt is the number of shards. -func (p *LogProject) CreateLogStore(name string, ttl, shardCnt int) (err error) { - - type Body struct { - Name string `json:"logstoreName"` - TTL int `json:"ttl"` - ShardCount int `json:"shardCount"` - } - - store := &Body{ - Name: name, - TTL: ttl, - ShardCount: shardCnt, - } - - body, err := json.Marshal(store) - if err != nil { - return - } - - h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), - "Content-Type": "application/json", - "Accept-Encoding": "deflate", // TODO: support lz4 - } - - r, err := request(p, "POST", "/logstores", h, body) - if err != nil { - return - } - - body, err = ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to create logstore") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - return -} - -// DeleteLogStore deletes a logstore according by logstore name. -func (p *LogProject) DeleteLogStore(name string) (err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - r, err := request(p, "DELETE", "/logstores/"+name, h, nil) - if err != nil { - return - } - - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to delete logstore") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - return -} - -// UpdateLogStore updates a logstore according by logstore name, -// obviously we can't modify the logstore name itself. -func (p *LogProject) UpdateLogStore(name string, ttl, shardCnt int) (err error) { - - type Body struct { - Name string `json:"logstoreName"` - TTL int `json:"ttl"` - ShardCount int `json:"shardCount"` - } - - store := &Body{ - Name: name, - TTL: ttl, - ShardCount: shardCnt, - } - - body, err := json.Marshal(store) - if err != nil { - return - } - - h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), - "Content-Type": "application/json", - "Accept-Encoding": "deflate", // TODO: support lz4 - } - - r, err := request(p, "PUT", "/logstores", h, body) - if err != nil { - return - } - - body, err = ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to update logstore") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - return -} - -// ListMachineGroup returns machine group name list and the total number of machine groups. -// The offset starts from 0 and the size is the max number of machine groups could be returned. -func (p *LogProject) ListMachineGroup(offset, size int) (m []string, total int, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - if size <= 0 { - size = 500 - } - - uri := fmt.Sprintf("/machinegroups?offset=%v&size=%v", offset, size) - r, err := request(p, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to list machine group") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - type Body struct { - MachineGroups []string - Count int - Total int - } - body := &Body{} - - err = json.Unmarshal(buf, body) - if err != nil { - return - } - - m = body.MachineGroups - total = body.Total - - return -} - -// GetMachineGroup retruns machine group according by machine group name. -func (p *LogProject) GetMachineGroup(name string) (m *MachineGroup, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - r, err := request(p, "GET", "/machinegroups/"+name, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to get machine group:%v", name) - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - m = &MachineGroup{} - err = json.Unmarshal(buf, m) - if err != nil { - return - } - m.project = p - return -} - -// CreateMachineGroup creates a new machine group in SLS. -func (p *LogProject) CreateMachineGroup(m *MachineGroup) (err error) { - - body, err := json.Marshal(m) - if err != nil { - return - } - - h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), - "Content-Type": "application/json", - "Accept-Encoding": "deflate", // TODO: support lz4 - } - - r, err := request(p, "POST", "/machinegroups", h, body) - if err != nil { - return - } - - body, err = ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to create machine group") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - return -} - -// UpdateMachineGroup updates a machine group. -func (p *LogProject) UpdateMachineGroup(m *MachineGroup) (err error) { - - body, err := json.Marshal(m) - if err != nil { - return - } - - h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), - "Content-Type": "application/json", - "Accept-Encoding": "deflate", // TODO: support lz4 - } - - r, err := request(p, "PUT", "/machinegroups/"+m.Name, h, body) - if err != nil { - return - } - - body, err = ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to update machine group") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - return -} - -// DeleteMachineGroup deletes machine group according machine group name. -func (p *LogProject) DeleteMachineGroup(name string) (err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - r, err := request(p, "DELETE", "/machinegroups/"+name, h, nil) - if err != nil { - return - } - - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to delete machine group") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - return -} - -// ListConfig returns config names list and the total number of configs. -// The offset starts from 0 and the size is the max number of configs could be returned. -func (p *LogProject) ListConfig(offset, size int) (cfgNames []string, total int, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - if size <= 0 { - size = 100 - } - - uri := fmt.Sprintf("/configs?offset=%v&size=%v", offset, size) - r, err := request(p, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to delete machine group") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - type Body struct { - Total int - Configs []string - } - body := &Body{} - - err = json.Unmarshal(buf, body) - if err != nil { - return - } - - cfgNames = body.Configs - total = body.Total - return -} - -// GetConfig returns config according by config name. -func (p *LogProject) GetConfig(name string) (c *LogConfig, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - r, err := request(p, "GET", "/configs/"+name, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to delete config") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - c = &LogConfig{} - err = json.Unmarshal(buf, c) - if err != nil { - return - } - c.project = p - return -} - -// UpdateConfig updates a config. -func (p *LogProject) UpdateConfig(c *LogConfig) (err error) { - - body, err := json.Marshal(c) - if err != nil { - return - } - - h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), - "Content-Type": "application/json", - "Accept-Encoding": "deflate", // TODO: support lz4 - } - - r, err := request(p, "PUT", "/configs/"+c.Name, h, body) - if err != nil { - return - } - - body, err = ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to update config") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - return -} - -// CreateConfig creates a new config in SLS. -func (p *LogProject) CreateConfig(c *LogConfig) (err error) { - - body, err := json.Marshal(c) - if err != nil { - return - } - - h := map[string]string{ - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), - "Content-Type": "application/json", - "Accept-Encoding": "deflate", // TODO: support lz4 - } - - r, err := request(p, "POST", "/configs", h, body) - if err != nil { - return - } - - body, err = ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to update config") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - return -} - -// DeleteConfig deletes a config according by config name. -func (p *LogProject) DeleteConfig(name string) (err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - r, err := request(p, "DELETE", "/configs/"+name, h, nil) - if err != nil { - return - } - - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(body, errMsg) - if err != nil { - err = fmt.Errorf("failed to delete config") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - return -} - -// GetAppliedMachineGroups returns applied machine group names list according config name. -func (p *LogProject) GetAppliedMachineGroups(confName string) (groupNames []string, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - uri := fmt.Sprintf("/configs/%v/machinegroups", confName) - r, err := request(p, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to get applied machine groups") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - type Body struct { - Count int - Machinegroups []string - } - - body := &Body{} - err = json.Unmarshal(buf, body) - if err != nil { - return - } - - groupNames = body.Machinegroups - return -} - -// GetAppliedConfigs returns applied config names list according machine group name groupName. -func (p *LogProject) GetAppliedConfigs(groupName string) (confNames []string, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - uri := fmt.Sprintf("/machinegroups/%v/configs", groupName) - r, err := request(p, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to applied configs") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - type Cfg struct { - Count int `json:"count"` - Configs []string `json:"configs"` - } - - body := &Cfg{} - err = json.Unmarshal(buf, body) - if err != nil { - return - } - - confNames = body.Configs - return -} - -// ApplyConfigToMachineGroup applies config to machine group. -func (p *LogProject) ApplyConfigToMachineGroup(confName, groupName string) (err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - uri := fmt.Sprintf("/machinegroups/%v/configs/%v", groupName, confName) - r, err := request(p, "PUT", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to apply config to machine group") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - return -} - -// RemoveConfigFromMachineGroup removes config from machine group. -func (p *LogProject) RemoveConfigFromMachineGroup(confName, groupName string) (err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - uri := fmt.Sprintf("/machinegroups/%v/configs/%v", groupName, confName) - r, err := request(p, "DELETE", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to remove config from machine group") - dump, _ := httputil.DumpResponse(r, true) - fmt.Printf("%s\n", dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - return -} diff --git a/logs/alils/log_store.go b/logs/alils/log_store.go deleted file mode 100755 index fa50273..0000000 --- a/logs/alils/log_store.go +++ /dev/null @@ -1,271 +0,0 @@ -package alils - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httputil" - "strconv" - - lz4 "github.com/cloudflare/golz4" - "github.com/gogo/protobuf/proto" -) - -// LogStore Store the logs -type LogStore struct { - Name string `json:"logstoreName"` - TTL int - ShardCount int - - CreateTime uint32 - LastModifyTime uint32 - - project *LogProject -} - -// Shard define the Log Shard -type Shard struct { - ShardID int `json:"shardID"` -} - -// ListShards returns shard id list of this logstore. -func (s *LogStore) ListShards() (shardIDs []int, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - uri := fmt.Sprintf("/logstores/%v/shards", s.Name) - r, err := request(s.project, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to list logstore") - dump, _ := httputil.DumpResponse(r, true) - fmt.Println(dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - var shards []*Shard - err = json.Unmarshal(buf, &shards) - if err != nil { - return - } - - for _, v := range shards { - shardIDs = append(shardIDs, v.ShardID) - } - return -} - -// PutLogs put logs into logstore. -// The callers should transform user logs into LogGroup. -func (s *LogStore) PutLogs(lg *LogGroup) (err error) { - body, err := proto.Marshal(lg) - if err != nil { - return - } - - // Compresse body with lz4 - out := make([]byte, lz4.CompressBound(body)) - n, err := lz4.Compress(body, out) - if err != nil { - return - } - - h := map[string]string{ - "x-sls-compresstype": "lz4", - "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), - "Content-Type": "application/x-protobuf", - } - - uri := fmt.Sprintf("/logstores/%v", s.Name) - r, err := request(s.project, "POST", uri, h, out[:n]) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to put logs") - dump, _ := httputil.DumpResponse(r, true) - fmt.Println(dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - return -} - -// GetCursor gets log cursor of one shard specified by shardID. -// The from can be in three form: a) unix timestamp in seccond, b) "begin", c) "end". -// For more detail please read: http://gitlab.alibaba-inc.com/sls/doc/blob/master/api/shard.md#logstore -func (s *LogStore) GetCursor(shardID int, from string) (cursor string, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - uri := fmt.Sprintf("/logstores/%v/shards/%v?type=cursor&from=%v", - s.Name, shardID, from) - - r, err := request(s.project, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to get cursor") - dump, _ := httputil.DumpResponse(r, true) - fmt.Println(dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - type Body struct { - Cursor string - } - body := &Body{} - - err = json.Unmarshal(buf, body) - if err != nil { - return - } - cursor = body.Cursor - return -} - -// GetLogsBytes gets logs binary data from shard specified by shardID according cursor. -// The logGroupMaxCount is the max number of logGroup could be returned. -// The nextCursor is the next curosr can be used to read logs at next time. -func (s *LogStore) GetLogsBytes(shardID int, cursor string, - logGroupMaxCount int) (out []byte, nextCursor string, err error) { - - h := map[string]string{ - "x-sls-bodyrawsize": "0", - "Accept": "application/x-protobuf", - "Accept-Encoding": "lz4", - } - - uri := fmt.Sprintf("/logstores/%v/shards/%v?type=logs&cursor=%v&count=%v", - s.Name, shardID, cursor, logGroupMaxCount) - - r, err := request(s.project, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to get cursor") - dump, _ := httputil.DumpResponse(r, true) - fmt.Println(dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - v, ok := r.Header["X-Sls-Compresstype"] - if !ok || len(v) == 0 { - err = fmt.Errorf("can't find 'x-sls-compresstype' header") - return - } - if v[0] != "lz4" { - err = fmt.Errorf("unexpected compress type:%v", v[0]) - return - } - - v, ok = r.Header["X-Sls-Cursor"] - if !ok || len(v) == 0 { - err = fmt.Errorf("can't find 'x-sls-cursor' header") - return - } - nextCursor = v[0] - - v, ok = r.Header["X-Sls-Bodyrawsize"] - if !ok || len(v) == 0 { - err = fmt.Errorf("can't find 'x-sls-bodyrawsize' header") - return - } - bodyRawSize, err := strconv.Atoi(v[0]) - if err != nil { - return - } - - out = make([]byte, bodyRawSize) - err = lz4.Uncompress(buf, out) - if err != nil { - return - } - - return -} - -// LogsBytesDecode decodes logs binary data retruned by GetLogsBytes API -func LogsBytesDecode(data []byte) (gl *LogGroupList, err error) { - - gl = &LogGroupList{} - err = proto.Unmarshal(data, gl) - if err != nil { - return - } - - return -} - -// GetLogs gets logs from shard specified by shardID according cursor. -// The logGroupMaxCount is the max number of logGroup could be returned. -// The nextCursor is the next curosr can be used to read logs at next time. -func (s *LogStore) GetLogs(shardID int, cursor string, - logGroupMaxCount int) (gl *LogGroupList, nextCursor string, err error) { - - out, nextCursor, err := s.GetLogsBytes(shardID, cursor, logGroupMaxCount) - if err != nil { - return - } - - gl, err = LogsBytesDecode(out) - if err != nil { - return - } - - return -} diff --git a/logs/alils/machine_group.go b/logs/alils/machine_group.go deleted file mode 100755 index b6c69a1..0000000 --- a/logs/alils/machine_group.go +++ /dev/null @@ -1,91 +0,0 @@ -package alils - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "net/http/httputil" -) - -// MachineGroupAttribute define the Attribute -type MachineGroupAttribute struct { - ExternalName string `json:"externalName"` - TopicName string `json:"groupTopic"` -} - -// MachineGroup define the machine Group -type MachineGroup struct { - Name string `json:"groupName"` - Type string `json:"groupType"` - MachineIDType string `json:"machineIdentifyType"` - MachineIDList []string `json:"machineList"` - - Attribute MachineGroupAttribute `json:"groupAttribute"` - - CreateTime uint32 - LastModifyTime uint32 - - project *LogProject -} - -// Machine define the Machine -type Machine struct { - IP string - UniqueID string `json:"machine-uniqueid"` - UserdefinedID string `json:"userdefined-id"` -} - -// MachineList define the Machine List -type MachineList struct { - Total int - Machines []*Machine -} - -// ListMachines returns machine list of this machine group. -func (m *MachineGroup) ListMachines() (ms []*Machine, total int, err error) { - h := map[string]string{ - "x-sls-bodyrawsize": "0", - } - - uri := fmt.Sprintf("/machinegroups/%v/machines", m.Name) - r, err := request(m.project, "GET", uri, h, nil) - if err != nil { - return - } - - buf, err := ioutil.ReadAll(r.Body) - if err != nil { - return - } - - if r.StatusCode != http.StatusOK { - errMsg := &errorMessage{} - err = json.Unmarshal(buf, errMsg) - if err != nil { - err = fmt.Errorf("failed to remove config from machine group") - dump, _ := httputil.DumpResponse(r, true) - fmt.Println(dump) - return - } - err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) - return - } - - body := &MachineList{} - err = json.Unmarshal(buf, body) - if err != nil { - return - } - - ms = body.Machines - total = body.Total - - return -} - -// GetAppliedConfigs returns applied configs of this machine group. -func (m *MachineGroup) GetAppliedConfigs() (confNames []string, err error) { - confNames, err = m.project.GetAppliedConfigs(m.Name) - return -} diff --git a/logs/alils/request.go b/logs/alils/request.go deleted file mode 100755 index 50d9c43..0000000 --- a/logs/alils/request.go +++ /dev/null @@ -1,62 +0,0 @@ -package alils - -import ( - "bytes" - "crypto/md5" - "fmt" - "net/http" -) - -// request sends a request to SLS. -func request(project *LogProject, method, uri string, headers map[string]string, - body []byte) (resp *http.Response, err error) { - - // The caller should provide 'x-sls-bodyrawsize' header - if _, ok := headers["x-sls-bodyrawsize"]; !ok { - err = fmt.Errorf("Can't find 'x-sls-bodyrawsize' header") - return - } - - // SLS public request headers - headers["Host"] = project.Name + "." + project.Endpoint - headers["Date"] = nowRFC1123() - headers["x-sls-apiversion"] = version - headers["x-sls-signaturemethod"] = signatureMethod - if body != nil { - bodyMD5 := fmt.Sprintf("%X", md5.Sum(body)) - headers["Content-MD5"] = bodyMD5 - - if _, ok := headers["Content-Type"]; !ok { - err = fmt.Errorf("Can't find 'Content-Type' header") - return - } - } - - // Calc Authorization - // Authorization = "SLS :" - digest, err := signature(project, method, uri, headers) - if err != nil { - return - } - auth := fmt.Sprintf("SLS %v:%v", project.AccessKeyID, digest) - headers["Authorization"] = auth - - // Initialize http request - reader := bytes.NewReader(body) - urlStr := fmt.Sprintf("http://%v.%v%v", project.Name, project.Endpoint, uri) - req, err := http.NewRequest(method, urlStr, reader) - if err != nil { - return - } - for k, v := range headers { - req.Header.Add(k, v) - } - - // Get ready to do request - resp, err = http.DefaultClient.Do(req) - if err != nil { - return - } - - return -} diff --git a/logs/alils/signature.go b/logs/alils/signature.go deleted file mode 100755 index 2d61130..0000000 --- a/logs/alils/signature.go +++ /dev/null @@ -1,111 +0,0 @@ -package alils - -import ( - "crypto/hmac" - "crypto/sha1" - "encoding/base64" - "fmt" - "net/url" - "sort" - "strings" - "time" -) - -// GMT location -var gmtLoc = time.FixedZone("GMT", 0) - -// NowRFC1123 returns now time in RFC1123 format with GMT timezone, -// eg. "Mon, 02 Jan 2006 15:04:05 GMT". -func nowRFC1123() string { - return time.Now().In(gmtLoc).Format(time.RFC1123) -} - -// signature calculates a request's signature digest. -func signature(project *LogProject, method, uri string, - headers map[string]string) (digest string, err error) { - var contentMD5, contentType, date, canoHeaders, canoResource string - var slsHeaderKeys sort.StringSlice - - // SignString = VERB + "\n" - // + CONTENT-MD5 + "\n" - // + CONTENT-TYPE + "\n" - // + DATE + "\n" - // + CanonicalizedSLSHeaders + "\n" - // + CanonicalizedResource - - if val, ok := headers["Content-MD5"]; ok { - contentMD5 = val - } - - if val, ok := headers["Content-Type"]; ok { - contentType = val - } - - date, ok := headers["Date"] - if !ok { - err = fmt.Errorf("Can't find 'Date' header") - return - } - - // Calc CanonicalizedSLSHeaders - slsHeaders := make(map[string]string, len(headers)) - for k, v := range headers { - l := strings.TrimSpace(strings.ToLower(k)) - if strings.HasPrefix(l, "x-sls-") { - slsHeaders[l] = strings.TrimSpace(v) - slsHeaderKeys = append(slsHeaderKeys, l) - } - } - - sort.Sort(slsHeaderKeys) - for i, k := range slsHeaderKeys { - canoHeaders += k + ":" + slsHeaders[k] - if i+1 < len(slsHeaderKeys) { - canoHeaders += "\n" - } - } - - // Calc CanonicalizedResource - u, err := url.Parse(uri) - if err != nil { - return - } - - canoResource += url.QueryEscape(u.Path) - if u.RawQuery != "" { - var keys sort.StringSlice - - vals := u.Query() - for k := range vals { - keys = append(keys, k) - } - - sort.Sort(keys) - canoResource += "?" - for i, k := range keys { - if i > 0 { - canoResource += "&" - } - - for _, v := range vals[k] { - canoResource += k + "=" + v - } - } - } - - signStr := method + "\n" + - contentMD5 + "\n" + - contentType + "\n" + - date + "\n" + - canoHeaders + "\n" + - canoResource - - // Signature = base64(hmac-sha1(UTF8-Encoding-Of(SignString)，AccessKeySecret)) - mac := hmac.New(sha1.New, []byte(project.AccessKeySecret)) - _, err = mac.Write([]byte(signStr)) - if err != nil { - return - } - digest = base64.StdEncoding.EncodeToString(mac.Sum(nil)) - return -} diff --git a/logs/alils/sys.go b/logs/alils/sys.go new file mode 100644 index 0000000..a62f526 --- /dev/null +++ b/logs/alils/sys.go @@ -0,0 +1,47 @@ +package alils + +import ( + "fmt" + "net" + "os" +) + +func GetCurrentInterface() *net.Interface { + if inter, err := net.InterfaceByName("eth0"); err == nil { + return inter + } else if inter, err := net.InterfaceByName("en0"); err == nil { + return inter + } + return nil +} + +func GetCurrentInterfaceHardwareAddr() string { + if inter := GetCurrentInterface(); inter != nil { + return fmt.Sprintf("%s", inter.HardwareAddr) + } + return "" +} + +func GetCurrentInterfaceAddrs() string { + if inter := GetCurrentInterface(); inter != nil { + if addrs, err := inter.Addrs(); err == nil { + return fmt.Sprintf("%s", addrs) + } + } + return "" +} + +func GetHostname() string { + hostname, _ := os.Hostname() + return hostname +} + +func GetUserHomename() string { + homedir, _ := os.UserHomeDir() + return homedir +} + +func Getwd() string { + wd, _ := os.Getwd() + return wd +} diff --git a/options.go b/options.go index 887e8c7..38a6084 100644 --- a/options.go +++ b/options.go @@ -21,4 +21,5 @@ const ( AdapterConsole Adapter = "console" AdapterSocket Adapter = "socket" AdapterElasticSearch Adapter = "es" + AdapterAliLs Adapter = "alils" )