feat: alerting for disk used
This commit is contained in:
parent
12bcc9ac57
commit
a8c0b3ffbb
39
cmd/main.go
39
cmd/main.go
@ -1,6 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"go/slack-bot/pkg/check"
|
||||
@ -19,6 +21,7 @@ type SlackConf struct {
|
||||
BotID string `hcl:"bot_id"`
|
||||
DestChannelID string `hcl:"dest_channel_id"`
|
||||
WebhookID string `hcl:"webhook_id"`
|
||||
DryRun *bool `hcl:"dry_run"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
@ -29,10 +32,40 @@ func main() {
|
||||
conf := ParseConfiguration(configFile)
|
||||
slackNotifier := notify.NewSlackNotifier(conf.BotID, conf.DestChannelID, conf.WebhookID)
|
||||
|
||||
msg := check.NewCliChecker().
|
||||
Check()
|
||||
dfChecker := check.NewDfChecker("disk is full", "bash", "-c", "df | grep -v 'auto_home'")
|
||||
|
||||
slackNotifier.Notify(msg)
|
||||
msg := dfChecker.Check(func(stdout string) notify.Message {
|
||||
parsedDf, err := dfChecker.Parse(stdout)
|
||||
if err != nil {
|
||||
return notify.Message{
|
||||
Content: fmt.Sprint("disk is full : ", err),
|
||||
ShouldNotify: true,
|
||||
}
|
||||
} else {
|
||||
casted := parsedDf.(check.DfOut)
|
||||
shouldNotify := check.FileSystemChecker("devfs", 70, casted)
|
||||
if shouldNotify {
|
||||
j, err := json.MarshalIndent(casted, "", " ")
|
||||
if err != nil {
|
||||
return notify.Message{
|
||||
Content: fmt.Sprint("disk is full : ", err),
|
||||
ShouldNotify: true,
|
||||
}
|
||||
}
|
||||
return notify.Message{
|
||||
Content: fmt.Sprintf("disk is full : ```%s```\n", j),
|
||||
ShouldNotify: true,
|
||||
}
|
||||
} else {
|
||||
return notify.Message{
|
||||
ShouldNotify: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
dryRunCtx := context.WithValue(context.Background(), notify.DryRunContextKey, conf.DryRun)
|
||||
slackNotifier.Notify(dryRunCtx, msg)
|
||||
}
|
||||
|
||||
//ParseConfiguration take filename path and parse it to Config.SlackConf
|
||||
|
@ -1,24 +1,51 @@
|
||||
package check
|
||||
|
||||
import "go/slack-bot/pkg/notify"
|
||||
import (
|
||||
"fmt"
|
||||
"go/slack-bot/pkg/notify"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
//Checker representation of check set
|
||||
type Checker interface {
|
||||
Check() notify.Message
|
||||
Check(f func(stdout string) notify.Message) notify.Message
|
||||
}
|
||||
|
||||
//AdvancedChecker like Checker except contain Parsing step
|
||||
type AdvancedChecker interface {
|
||||
Checker
|
||||
Parse(stdout string) (interface{}, error)
|
||||
}
|
||||
|
||||
//CliChecker implement checker with cli command
|
||||
type CliChecker struct {
|
||||
alertingMessage string
|
||||
command []string
|
||||
}
|
||||
|
||||
//NewCliChecker buld new cli checker
|
||||
func NewCliChecker() *CliChecker {
|
||||
return &CliChecker{}
|
||||
//NewCliChecker build new cli checker
|
||||
func NewCliChecker(alertingMessage string, command ...string) Checker {
|
||||
return &CliChecker{
|
||||
alertingMessage: alertingMessage,
|
||||
command: command,
|
||||
}
|
||||
}
|
||||
|
||||
//Check function use to run cli call to check some parameter
|
||||
func (cc *CliChecker) Check() notify.Message {
|
||||
return notify.Message{
|
||||
Content: "full disk",
|
||||
ShouldNotify: true,
|
||||
func (cc *CliChecker) Check(f func(stdout string) notify.Message) notify.Message {
|
||||
message := notify.Message{ShouldNotify: false}
|
||||
|
||||
out, err := exec.Command(cc.command[0], cc.command[1:]...).CombinedOutput()
|
||||
if err != nil {
|
||||
// command fail notify with error
|
||||
message = notify.Message{
|
||||
Content: fmt.Sprint(cc.alertingMessage, " : ", string(out), err),
|
||||
ShouldNotify: true,
|
||||
}
|
||||
} else {
|
||||
// command success check if condition required notify
|
||||
message = f(string(out))
|
||||
}
|
||||
|
||||
return message
|
||||
}
|
||||
|
80
pkg/check/df.go
Normal file
80
pkg/check/df.go
Normal file
@ -0,0 +1,80 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
//DfChecker df command implementation of AdvancedChecker
|
||||
type DfChecker struct {
|
||||
CliChecker
|
||||
pourcentRegexMatcher *regexp.Regexp
|
||||
}
|
||||
|
||||
type DfOut []DfOutLine
|
||||
type DfOutLine struct {
|
||||
Filesystem,
|
||||
Size,
|
||||
Used,
|
||||
Available string
|
||||
PourcentUsed int
|
||||
Mounted string
|
||||
}
|
||||
|
||||
//NewDfChecker build new checker for df command
|
||||
func NewDfChecker(alertingMessage string, command ...string) AdvancedChecker {
|
||||
|
||||
return &DfChecker{
|
||||
CliChecker: CliChecker{
|
||||
alertingMessage: alertingMessage,
|
||||
command: command,
|
||||
},
|
||||
pourcentRegexMatcher: regexp.MustCompile(`(?P<value>.*)%`),
|
||||
}
|
||||
}
|
||||
|
||||
//DfParser parse df stdout
|
||||
func (dc *DfChecker) Parse(stdout string) (interface{}, error) {
|
||||
var dfOut DfOut
|
||||
|
||||
for _, line := range strings.Split(stdout, "\n")[1:] {
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
var cleanedColumns []string
|
||||
for _, col := range strings.Split(line, " ") {
|
||||
if v := strings.TrimSpace(col); v != "" {
|
||||
cleanedColumns = append(cleanedColumns, v)
|
||||
}
|
||||
}
|
||||
|
||||
valueIndex := dc.pourcentRegexMatcher.SubexpIndex("value")
|
||||
v, err := strconv.Atoi(dc.pourcentRegexMatcher.FindStringSubmatch(cleanedColumns[4])[valueIndex])
|
||||
if err != nil {
|
||||
return dfOut, err
|
||||
}
|
||||
// extract pourcent used from columns
|
||||
dfOut = append(dfOut, DfOutLine{
|
||||
Filesystem: cleanedColumns[0],
|
||||
Size: cleanedColumns[1],
|
||||
Used: cleanedColumns[2],
|
||||
Available: cleanedColumns[3],
|
||||
PourcentUsed: v,
|
||||
Mounted: cleanedColumns[len(cleanedColumns)-1],
|
||||
})
|
||||
}
|
||||
return dfOut, nil
|
||||
}
|
||||
|
||||
func FileSystemChecker(fileSystemName string, criticalPourcent int, dfOut DfOut) bool {
|
||||
for _, line := range dfOut {
|
||||
if line.Filesystem == fileSystemName {
|
||||
if line.PourcentUsed > criticalPourcent {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
@ -1,13 +1,19 @@
|
||||
package notify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
)
|
||||
|
||||
const slackHookBaseURL = "hooks.slack.com/services"
|
||||
type SlackContextKey string
|
||||
|
||||
const (
|
||||
slackHookBaseURL = "hooks.slack.com/services"
|
||||
DryRunContextKey SlackContextKey = "dryRun"
|
||||
)
|
||||
|
||||
//Notifier interface represent mean to send notification
|
||||
type Notifier interface {
|
||||
@ -36,16 +42,21 @@ func NewSlackNotifier(BotID, DestChannelID, WebhookID string) *SlackNotifier {
|
||||
}
|
||||
|
||||
//Notify send notification to channel
|
||||
func (sn *SlackNotifier) Notify(message Message) {
|
||||
func (sn *SlackNotifier) Notify(ctx context.Context, message Message) {
|
||||
if message.ShouldNotify {
|
||||
msg := &slack.WebhookMessage{
|
||||
Text: message.Content,
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://%s/%s/%s/%s", slackHookBaseURL, sn.BotID, sn.DestChannelID, sn.WebhookID)
|
||||
err := slack.PostWebhook(url, msg)
|
||||
if err != nil {
|
||||
log.Printf("Send message error %+v", err)
|
||||
if v := ctx.Value(DryRunContextKey); v != nil && v == true {
|
||||
log.Printf("dryrun mode : slack notify to %s\n", url)
|
||||
log.Printf("dryrun mode : %+v\n", msg)
|
||||
} else {
|
||||
err := slack.PostWebhook(url, msg)
|
||||
if err != nil {
|
||||
log.Printf("Send message error %+v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user