From 7f556de2d4e7114d2a39a5ddd216a3b3c0a02fe2 Mon Sep 17 00:00:00 2001 From: RouxAntoine Date: Thu, 11 Mar 2021 02:59:21 +0100 Subject: [PATCH] poller build with dockerfile static binary --- .dockerignore | 10 ++++ .vscode/launch.json | 2 +- Dockerfile | 57 ++++++++++++++++++++++ Makefile | 27 +++++++++-- cmd/poller/main.go | 105 ++++++++++++++++++++--------------------- config.sample.hcl | 10 +++- go.mod | 3 +- go.sum | 27 +++++++++-- internal/config.go | 55 ++++++++++++++++++++- internal/storage/s3.go | 48 ++++++++++++++----- 10 files changed, 265 insertions(+), 79 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f4825e5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +bin/ +.vscode/ +.git/ +certs/out/ +!certs/out/.gitkeep +db.json +*.log +config.hcl +Dockerfile +.gitignore \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index f2669d7..6113295 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,7 @@ "mode": "auto", "program": "${workspaceFolder}/cmd/weather/main.go", "cwd": "${workspaceFolder}" - } + }, { "name": "Run poller applicaton", "type": "go", diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..df5d04f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,57 @@ +FROM --platform=$BUILDPLATFORM golang:1.16.0-alpine3.13 AS builder +# ex. amd64 +ARG TARGETARCH +# ex. linux/amd64 +ARG BUILDPLATFORM + +RUN apk update && \ + apk add --no-cache git ca-certificates make tzdata gcc libc-dev && \ + apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing cfssl && \ + update-ca-certificates + +ENV USER=appuser +ENV UID=10001 +ENV GID=10001 +WORKDIR /data + +RUN addgroup \ + --gid "${GID}" \ + "${USER}" && \ + adduser \ + --disabled-password \ + --gecos "" \ + --home "/nonexistent" \ + --shell "/sbin/nologin" \ + --no-create-home \ + --uid "${UID}" \ + -G "${USER}" \ + "${USER}" + +COPY Makefile . +COPY certs/ certs/ +RUN make gen-cert + +COPY go.mod . +COPY go.sum . +RUN make dependencies + +ARG BINARYNAME="poller" +# ARG BINARYNAME="weather" + +COPY . . +# RUN make build-$BINARYNAME GOARCH=$TARGETARCH GOOS=$(echo $BUILDPLATFORM | cut -d'/' -f1) LDFLAGS='-extldflags="-static"' +RUN make build-$BINARYNAME GOARCH=$TARGETARCH GOOS=$(echo $BUILDPLATFORM | cut -d'/' -f1) GOBUILDFLAGS="-a -tags netgo -installsuffix netgo" + +FROM scratch + +USER appuser:appuser + +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /etc/passwd /etc/passwd +COPY --from=builder /etc/group /etc/group +COPY --from=builder --chown=appuser:appuser /data/bin/* /go/bin/poller +COPY --from=builder --chown=appuser:appuser /data/certs/out/weather* /go + +WORKDIR /go +ENTRYPOINT ["/go/bin/poller"] diff --git a/Makefile b/Makefile index 2610def..ca6ba8b 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,32 @@ .PHONY: gen-cert build-weather build-poller gen-cert dev-dependencies cert-info cert-info-remote curl-test clean get-root-ca +.EXPORT_ALL_VARIABLES: + +GOARCH=amd64 +# GOARCH=arm +GOOS=linux +# GOOS=darwin +LDFLAGS=-w -s -d +GOBUILDFLAGS= +DOCKER_BUILDKIT=1 +CGO_ENABLED=0 build: build-poller build-weather -build-weather: - go build -o bin/weather cmd/weather/main.go +build-weather: dependencies + @echo "build for os $$GOOS and arch $$GOARCH" + go build -o bin/weather-$(GOOS)-$(GOARCH) cmd/weather/main.go -build-poller: - go build -o bin/poller cmd/poller/main.go +build-poller: dependencies + @echo "build for os $$GOOS and arch $$GOARCH" + go build -o bin/poller-$(GOOS)-$(GOARCH) -ldflags="$(LDFLAGS)" $(GOBUILDFLAGS) cmd/poller/main.go gen-cert: cfssl gencert -config certs/client-config.json -profile server -hostname weather.localdomain certs/client-csr.json | cfssljson -bare certs/out/weather +dependencies: + go mod download + go mod verify + dev-dependencies: go get github.com/cloudflare/cfssl/cmd/cfssl go get github.com/cloudflare/cfssl/cmd/cfssljson @@ -30,3 +46,6 @@ clean: get-root-ca: cfssl info -remote rasp1.localdomain:444 -config certs/client-config.json | cfssljson -bare -stdout /dev/stdout | tee certs/out/ca.pem + +docker-build: + docker build -t docker.registry:5000/weather . \ No newline at end of file diff --git a/cmd/poller/main.go b/cmd/poller/main.go index 85a1034..d2eba8d 100644 --- a/cmd/poller/main.go +++ b/cmd/poller/main.go @@ -1,91 +1,46 @@ package main import ( - "bytes" "flag" - "fmt" "go/weather/internal" "go/weather/internal/poller" "go/weather/internal/storage" "go/weather/internal/web" "go/weather/pkg/logger" - "io" - "io/ioutil" "os" "os/signal" + "strings" "time" - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/gohcl" - "github.com/hashicorp/hcl/v2/hclsyntax" - "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) const bucket = "weather" func main() { - var interval time.Duration - var filename string - flag.DurationVar(&interval, "check-interval", time.Second*30, "the sleep duration between multiple call to remote weather api") - flag.StringVar(&filename, "filename", "config.hcl", "configuration filename") - flag.Parse() + f := flagDefinition() - loggerLevel := zap.NewAtomicLevelAt(zap.DebugLevel) - defaultLogger := logger.NewLogger("poller", "weather.log", loggerLevel) - defaultLoggerSugar := defaultLogger.Sugar() + loggerLevel := zap.NewAtomicLevelAt(f.logLevel) + defaultLogger := logger.NewLogger("poller", f.logOutput, loggerLevel) + sLogger := defaultLogger.Sugar() defer defaultLogger.Sync() - if filename == "config.hcl" { - dir, err := os.Getwd() - if err != nil { - defaultLoggerSugar.Fatal("Fail to determine current directory to load default ./config.hcl file") - } - filename = fmt.Sprintf("%s%c%s", dir, os.PathSeparator, filename) - } - - src, err := ioutil.ReadFile(filename) - if err != nil { - defaultLoggerSugar.Fatal("Missing required parameter : filename") - } - file, diags := hclsyntax.ParseConfig(src, filename, hcl.Pos{Line: 1, Column: 1}) - if diags.HasErrors() { - defaultLoggerSugar.Fatal("config parse", zap.Error(diags)) - } - - config := &internal.WeatherConfig{} - - diags = gohcl.DecodeBody(file.Body, nil, config) - if diags.HasErrors() { - defaultLoggerSugar.Fatal("config parse", zap.Error(diags)) - } - - if config.OpenweatherSecret == "" { - defaultLoggerSugar.Fatal("Missing required parameter : openweather-secret") - } + config := internal.ParseConfiguration(sLogger, f.filename) addr := web.NewListenAddr("api.openweathermap.org", 443) poller := poller.NewWeatherPoller(defaultLogger, addr, 45.75, 4.85, config.OpenweatherSecret) - // s3storage := storage.NewS3Storage(defaultLogger, bucket) - fileStorage := storage.NewFileStorage(defaultLogger, "db.json") + s3storage := storage.NewS3Storage(defaultLogger, config.S3Storage, bucket) go func() { for { data := poller.Poll() defer data.Close() - - var buf bytes.Buffer - b, err := ioutil.ReadAll(io.TeeReader(data, &buf)) - if err != nil { - defaultLogger.Sugar().Fatal("Wait next poll") - } - - fileStorage.Store(bytes.NewReader(b)) - // s3storage.Store(&buf) + s3storage.Store(data) defaultLogger.Sugar().Debug("Wait next poll") - time.Sleep(interval) + time.Sleep(f.interval) } }() @@ -97,3 +52,43 @@ func main() { defaultLogger.Sugar().Info("shutting down") os.Exit(0) } + +//flagParameter contain all application customizable parameter +type flagParameter struct { + interval time.Duration + filename string + logLevel zapcore.Level + logOutput string +} + +func flagDefinition() flagParameter { + var logLevel string + f := flagParameter{} + flag.DurationVar(&f.interval, "check-interval", time.Second*30, "the sleep duration between multiple call to remote weather api") + flag.StringVar(&f.filename, "filename", "config.hcl", "configuration filename") + flag.StringVar(&logLevel, "logLevel", "info", "Log level") + flag.StringVar(&f.logOutput, "logOutput", "weather.log", "Output log path") + flag.Parse() + f.logLevel = parseLogLevel(logLevel) + + return f +} + +func parseLogLevel(logLevel string) zapcore.Level { + zapLevel := zap.InfoLevel + switch strings.ToUpper(logLevel) { + case "DEBUG": + zapLevel = zapcore.DebugLevel + case "INFO": + zapLevel = zapcore.InfoLevel + case "WARN": + zapLevel = zapcore.WarnLevel + case "ERROR": + zapLevel = zapcore.ErrorLevel + case "PANIC": + zapLevel = zapcore.PanicLevel + case "FATAL": + zapLevel = zapcore.FatalLevel + } + return zapLevel +} diff --git a/config.sample.hcl b/config.sample.hcl index d56da6e..6cded2c 100644 --- a/config.sample.hcl +++ b/config.sample.hcl @@ -1 +1,9 @@ -openweather-secret = "" \ No newline at end of file +openweather_secret = "" + +// this block is optional and override ~/.aws/config +s3 { + endpoint_url = "" + region = "" + aws_access_key_id = "" + aws_secret_access_key = "" +} \ No newline at end of file diff --git a/go.mod b/go.mod index 85008f1..c5698a7 100644 --- a/go.mod +++ b/go.mod @@ -2,13 +2,14 @@ module go/weather go 1.16 +replace github.com/hashicorp/hcl/v2 => antoine-roux.tk/projects/go/hcl/v2 v2.9.0-local-1 + require ( github.com/aws/aws-sdk-go v1.37.23 github.com/elastic/go-sysinfo v1.6.0 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-version v1.2.1 - github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/hcl/v2 v2.9.0 github.com/magefile/mage v1.11.0 // indirect github.com/pelletier/go-toml v1.8.1 diff --git a/go.sum b/go.sum index 37f904d..82a0d2a 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,10 @@ +antoine-roux.tk/projects/go/hcl/v2 v2.9.0-local-1 h1:GddyjZJMVd7GiF7lPVdBb2WRV+DoHIRm0pRyaSe8Jo8= +antoine-roux.tk/projects/go/hcl/v2 v2.9.0-local-1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3 h1:ZSTrOEhiM5J5RFxEaFvMZVEAM1KvT1YzbEOwB2EAGjA= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= @@ -26,20 +29,19 @@ github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl/v2 v2.9.0 h1:7kJiMiKBqGHASbDJuFAMlpRMJLyhuLg/IsU/3EzwniA= -github.com/hashicorp/hcl/v2 v2.9.0/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -47,9 +49,11 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -72,25 +76,33 @@ github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.0 h1:DMOzIV76tmoDNE9pX6RSN0aDtCYeCg5VueieJaAo1uw= github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1 h1:quXMXlA39OCbd2wAdTsGDlK9RkOk6Wuw+x37wVyIuWY= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/yuin/goldmark v1.1.27 h1:nqDD4MMMQA0lmWq03Z2/myGPYLQoXtmi0rGVs95ntbo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= go.elastic.co/apm v1.11.0 h1:uJyt6nCW9880sZhfl1tB//Jy/5TadNoAd8edRUtgb3w= go.elastic.co/apm v1.11.0/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0= @@ -110,6 +122,7 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -119,10 +132,12 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -137,6 +152,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -166,9 +182,12 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= diff --git a/internal/config.go b/internal/config.go index e4a7818..bdf0f62 100644 --- a/internal/config.go +++ b/internal/config.go @@ -1,7 +1,60 @@ package internal +import ( + "fmt" + "go/weather/internal/storage" + "io/ioutil" + "os" + "path/filepath" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclsyntax" + "go.uber.org/zap" +) + //WeatherConfig weather application relative configuration file // this configuration is common to weather server and poller type WeatherConfig struct { - OpenweatherSecret string `hcl:"openweather-secret"` + OpenweatherSecret string `hcl:"openweather_secret"` + S3Storage *storage.WeatherS3StorageConfig `hcl:"s3,block"` +} + +//ParseConfiguration parse configuration from filename path +func ParseConfiguration(sLogger *zap.SugaredLogger, filename string) *WeatherConfig { + + dir, err := os.Getwd() + if err != nil { + sLogger.Fatalw("Fail to determine current directory to load default ./config.hcl file", zap.Error(err)) + } + sLogger.Debugf("current dir is %s", dir) + + if filename == "config.hcl" { + filename = fmt.Sprintf("%s%c%s", dir, os.PathSeparator, filename) + } + relativeFilename, err := filepath.Rel(dir, filename) + if err != nil { + sLogger.Fatalw(fmt.Sprintf("Fail to determine configuration relative directory for file %s", filename), zap.Error(err)) + } + + src, err := ioutil.ReadFile(relativeFilename) + if err != nil { + sLogger.Fatalw(fmt.Sprintf("Missing required parameter filename got : %s", relativeFilename), zap.Error(err)) + } + file, diags := hclsyntax.ParseConfig(src, relativeFilename, hcl.Pos{Line: 1, Column: 1}) + if diags.HasErrors() { + sLogger.Fatal("config parse", zap.Error(diags)) + } + + config := &WeatherConfig{} + + diags = gohcl.DecodeBody(file.Body, nil, config) + if diags.HasErrors() { + sLogger.Fatal("config decode", zap.Error(diags)) + } + + if config.OpenweatherSecret == "" { + sLogger.Fatal("Missing required parameter : openweather-secret") + } + return config } diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 95ead2d..b76fcc6 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -8,6 +8,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/defaults" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -18,7 +19,7 @@ import ( //AwsDefaultSection toml default section type AwsDefaultSection struct { // attribute should be public ! for go-toml - EndpointURL string `toml:"endpoint_url"` + EndpointURL string `toml:"endpoint_url" hcl:"endpoint_url"` } //CustomAwsConfig custom toml config for aws @@ -26,6 +27,14 @@ type CustomAwsConfig struct { Default AwsDefaultSection `toml:"default"` } +//WeatherS3StorageConfig interface abstracting storageConfig +type WeatherS3StorageConfig struct { + AwsDefaultSection `hcl:",nested"` + Region string `hcl:"region"` + AwsAccessKeyID string `hcl:"aws_access_key_id"` + AwsSecretAccessKey string `hcl:"aws_secret_access_key"` +} + //s3Storage classe used to store oi.Reader to s3 type s3Storage struct { logger *logger.WeatherLogger @@ -34,23 +43,39 @@ type s3Storage struct { } //NewS3Storage instanciate storage object -func NewS3Storage(log *logger.WeatherLogger, bucket string) Storage { +func NewS3Storage(log *logger.WeatherLogger, config *WeatherS3StorageConfig, bucket string) Storage { s3 := s3Storage{ logger: log, bucket: bucket, } - customConfig := CustomAwsConfig{} - err := decodeFile(defaults.SharedConfigFilename(), &customConfig) - s3.handleError(err) + var consolidateConfig aws.Config + // when no config file arer specify overload weather configurable parameter with default one + if config == nil { + customConfig := CustomAwsConfig{} + err := decodeFile(defaults.SharedConfigFilename(), &customConfig) + s3.handleError(err) + + consolidateConfig = aws.Config{ + Endpoint: &customConfig.Default.EndpointURL, + DisableSSL: aws.Bool(true), + LogLevel: aws.LogLevel(aws.LogDebug), + S3ForcePathStyle: aws.Bool(true), + } + } else { + consolidateConfig = aws.Config{ + Endpoint: &config.EndpointURL, + Credentials: credentials.NewStaticCredentials(config.AwsAccessKeyID, config.AwsSecretAccessKey, ""), + Region: &config.Region, + DisableSSL: aws.Bool(true), + LogLevel: aws.LogLevel(aws.LogDebug), + S3ForcePathStyle: aws.Bool(true), + } + } s, err := session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, - Config: aws.Config{ - Endpoint: &customConfig.Default.EndpointURL, - DisableSSL: aws.Bool(true), - LogLevel: aws.LogLevel(aws.LogDebug), - }, + Config: consolidateConfig, }) s3.handleError(err) s3.session = s @@ -64,10 +89,9 @@ func (ss *s3Storage) Store(content io.Reader) { _, err := uploader.Upload(&s3manager.UploadInput{ Bucket: aws.String(ss.bucket), - Key: aws.String(time.Now().UTC().Format(time.RFC3339)), + Key: aws.String(fmt.Sprintf("%s.json", time.Now().UTC().Format(time.RFC3339))), Body: content, }) - fmt.Printf("failed to upload file, %v\n", err) ss.handleError(err) }