feat: alerting for disk used

This commit is contained in:
RouxAntoine 2021-03-22 03:39:42 +01:00
parent 12bcc9ac57
commit a8c0b3ffbb
Signed by: antoine
GPG Key ID: 098FB66FC0475E70
4 changed files with 168 additions and 17 deletions

View File

@ -1,6 +1,8 @@
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"go/slack-bot/pkg/check"
@ -19,6 +21,7 @@ type SlackConf struct {
BotID string `hcl:"bot_id"`
DestChannelID string `hcl:"dest_channel_id"`
WebhookID string `hcl:"webhook_id"`
DryRun *bool `hcl:"dry_run"`
}
func main() {
@ -29,10 +32,40 @@ func main() {
conf := ParseConfiguration(configFile)
slackNotifier := notify.NewSlackNotifier(conf.BotID, conf.DestChannelID, conf.WebhookID)
msg := check.NewCliChecker().
Check()
dfChecker := check.NewDfChecker("disk is full", "bash", "-c", "df | grep -v 'auto_home'")
slackNotifier.Notify(msg)
msg := dfChecker.Check(func(stdout string) notify.Message {
parsedDf, err := dfChecker.Parse(stdout)
if err != nil {
return notify.Message{
Content: fmt.Sprint("disk is full : ", err),
ShouldNotify: true,
}
} else {
casted := parsedDf.(check.DfOut)
shouldNotify := check.FileSystemChecker("devfs", 70, casted)
if shouldNotify {
j, err := json.MarshalIndent(casted, "", " ")
if err != nil {
return notify.Message{
Content: fmt.Sprint("disk is full : ", err),
ShouldNotify: true,
}
}
return notify.Message{
Content: fmt.Sprintf("disk is full : ```%s```\n", j),
ShouldNotify: true,
}
} else {
return notify.Message{
ShouldNotify: false,
}
}
}
})
dryRunCtx := context.WithValue(context.Background(), notify.DryRunContextKey, conf.DryRun)
slackNotifier.Notify(dryRunCtx, msg)
}
//ParseConfiguration take filename path and parse it to Config.SlackConf

View File

@ -1,24 +1,51 @@
package check
import "go/slack-bot/pkg/notify"
import (
"fmt"
"go/slack-bot/pkg/notify"
"os/exec"
)
//Checker representation of check set
type Checker interface {
Check() notify.Message
Check(f func(stdout string) notify.Message) notify.Message
}
//AdvancedChecker like Checker except contain Parsing step
type AdvancedChecker interface {
Checker
Parse(stdout string) (interface{}, error)
}
//CliChecker implement checker with cli command
type CliChecker struct {
alertingMessage string
command []string
}
//NewCliChecker buld new cli checker
func NewCliChecker() *CliChecker {
return &CliChecker{}
//NewCliChecker build new cli checker
func NewCliChecker(alertingMessage string, command ...string) Checker {
return &CliChecker{
alertingMessage: alertingMessage,
command: command,
}
}
//Check function use to run cli call to check some parameter
func (cc *CliChecker) Check() notify.Message {
return notify.Message{
Content: "full disk",
ShouldNotify: true,
func (cc *CliChecker) Check(f func(stdout string) notify.Message) notify.Message {
message := notify.Message{ShouldNotify: false}
out, err := exec.Command(cc.command[0], cc.command[1:]...).CombinedOutput()
if err != nil {
// command fail notify with error
message = notify.Message{
Content: fmt.Sprint(cc.alertingMessage, " : ", string(out), err),
ShouldNotify: true,
}
} else {
// command success check if condition required notify
message = f(string(out))
}
return message
}

80
pkg/check/df.go Normal file
View File

@ -0,0 +1,80 @@
package check
import (
"regexp"
"strconv"
"strings"
)
//DfChecker df command implementation of AdvancedChecker
type DfChecker struct {
CliChecker
pourcentRegexMatcher *regexp.Regexp
}
type DfOut []DfOutLine
type DfOutLine struct {
Filesystem,
Size,
Used,
Available string
PourcentUsed int
Mounted string
}
//NewDfChecker build new checker for df command
func NewDfChecker(alertingMessage string, command ...string) AdvancedChecker {
return &DfChecker{
CliChecker: CliChecker{
alertingMessage: alertingMessage,
command: command,
},
pourcentRegexMatcher: regexp.MustCompile(`(?P<value>.*)%`),
}
}
//DfParser parse df stdout
func (dc *DfChecker) Parse(stdout string) (interface{}, error) {
var dfOut DfOut
for _, line := range strings.Split(stdout, "\n")[1:] {
if strings.TrimSpace(line) == "" {
continue
}
var cleanedColumns []string
for _, col := range strings.Split(line, " ") {
if v := strings.TrimSpace(col); v != "" {
cleanedColumns = append(cleanedColumns, v)
}
}
valueIndex := dc.pourcentRegexMatcher.SubexpIndex("value")
v, err := strconv.Atoi(dc.pourcentRegexMatcher.FindStringSubmatch(cleanedColumns[4])[valueIndex])
if err != nil {
return dfOut, err
}
// extract pourcent used from columns
dfOut = append(dfOut, DfOutLine{
Filesystem: cleanedColumns[0],
Size: cleanedColumns[1],
Used: cleanedColumns[2],
Available: cleanedColumns[3],
PourcentUsed: v,
Mounted: cleanedColumns[len(cleanedColumns)-1],
})
}
return dfOut, nil
}
func FileSystemChecker(fileSystemName string, criticalPourcent int, dfOut DfOut) bool {
for _, line := range dfOut {
if line.Filesystem == fileSystemName {
if line.PourcentUsed > criticalPourcent {
return true
}
}
}
return false
}

View File

@ -1,13 +1,19 @@
package notify
import (
"context"
"fmt"
"log"
"github.com/slack-go/slack"
)
const slackHookBaseURL = "hooks.slack.com/services"
type SlackContextKey string
const (
slackHookBaseURL = "hooks.slack.com/services"
DryRunContextKey SlackContextKey = "dryRun"
)
//Notifier interface represent mean to send notification
type Notifier interface {
@ -36,16 +42,21 @@ func NewSlackNotifier(BotID, DestChannelID, WebhookID string) *SlackNotifier {
}
//Notify send notification to channel
func (sn *SlackNotifier) Notify(message Message) {
func (sn *SlackNotifier) Notify(ctx context.Context, message Message) {
if message.ShouldNotify {
msg := &slack.WebhookMessage{
Text: message.Content,
}
url := fmt.Sprintf("https://%s/%s/%s/%s", slackHookBaseURL, sn.BotID, sn.DestChannelID, sn.WebhookID)
err := slack.PostWebhook(url, msg)
if err != nil {
log.Printf("Send message error %+v", err)
if v := ctx.Value(DryRunContextKey); v != nil && v == true {
log.Printf("dryrun mode : slack notify to %s\n", url)
log.Printf("dryrun mode : %+v\n", msg)
} else {
err := slack.PostWebhook(url, msg)
if err != nil {
log.Printf("Send message error %+v", err)
}
}
}
}