diff --git a/.dockerignore b/.dockerignore index f4825e5..5811d5c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,5 @@ db.json *.log config.hcl Dockerfile -.gitignore \ No newline at end of file +.gitignore +manifests/ \ No newline at end of file diff --git a/DockerfilePoller b/DockerfilePoller index 8129832..904b232 100644 --- a/DockerfilePoller +++ b/DockerfilePoller @@ -32,12 +32,13 @@ COPY go.mod . COPY go.sum . RUN make dependencies +ARG POLLER_VERSION COPY . . RUN make build-poller \ GOARCH=$TARGETARCH \ GOOS=$(echo $BUILDPLATFORM | cut -d'/' -f1) \ - GOBUILDFLAGS="-a -tags netgo -installsuffix netgo" \ - LDFLAGS="-w -s -d" + GOBUILDFLAGS="-a -tags netgo,prod -installsuffix netgo" \ + LDFLAGS="-w -s -d -X go/weather/internal/version.Version=$POLLER_VERSION" FROM scratch diff --git a/DockerfileWeather b/DockerfileWeather index a92e42d..94d79ea 100644 --- a/DockerfileWeather +++ b/DockerfileWeather @@ -35,13 +35,14 @@ COPY go.mod . COPY go.sum . RUN make dependencies +ARG WEATHER_VERSION COPY . . # RUN make build-$BINARYNAME GOARCH=$TARGETARCH GOOS=$(echo $BUILDPLATFORM | cut -d'/' -f1) LDFLAGS='-extldflags="-static"' RUN make build-weather \ GOARCH=$TARGETARCH \ GOOS=$(echo $BUILDPLATFORM | cut -d'/' -f1) \ - GOBUILDFLAGS="-a -tags netgo -installsuffix netgo" \ - LDFLAGS="-w -s -d" + GOBUILDFLAGS=" -a -tags netgo,prod -installsuffix netgo" \ + LDFLAGS="-w -s -d -X go/weather/internal/version.Version=$WEATHER_VERSION" FROM alpine diff --git a/Makefile b/Makefile index 0c16411..0953f31 100644 --- a/Makefile +++ b/Makefile @@ -5,19 +5,21 @@ GOARCH=amd64 # GOARCH=arm GOOS=darwin # GOOS=linux -LDFLAGS=-w -s -GOBUILDFLAGS= -DOCKER_BUILDKIT=1 +LDFLAGS=-w -s -X go/weather/internal/version.Version=$$(git rev-list -1 HEAD) +GOBUILDFLAGS=-tags dev CGO_ENABLED=0 -WEATHER_VERSION=latest -POLLER_VERSION=latest -DOMAIN_ALIAS=weather.localdomain + +DOCKER_BUILDKIT=1 + +WEATHER_VERSION=1.0.0 +POLLER_VERSION=1.0.0 +DOMAIN_ALIAS=weather.localdomain,weather-dev.localdomain build: build-poller build-weather build-weather: dependencies @echo "build for os $$GOOS and arch $$GOARCH" - go build -o bin/weather-$(GOOS)-$(GOARCH) cmd/weather/main.go + go build -o bin/weather-$(GOOS)-$(GOARCH) -ldflags="$(LDFLAGS)" $(GOBUILDFLAGS) cmd/weather/main.go build-poller: dependencies @echo "build for os $$GOOS and arch $$GOARCH" @@ -50,15 +52,14 @@ clean: get-root-ca: cfssl info -remote rasp1.localdomain:444 -config certs/client-config.json | cfssljson -bare -stdout /dev/stdout | tee certs/out/ca.pem -docker: docker-build-poller docker-build-weather docker-push +docker: docker-build-poller docker-build-weather docker-build-poller: - docker build --force-rm -t docker.registry:5000/weather/poller:$(POLLER_VERSION) -f DockerfilePoller . -docker-build-weather: - docker build --force-rm -t docker.registry:5000/weather/server:$(WEATHER_VERSION) -f DockerfileWeather . - -docker-push: + docker build --force-rm -t docker.registry:5000/weather/poller:$(POLLER_VERSION) --build-arg POLLER_VERSION=$(POLLER_VERSION) -f DockerfilePoller . docker push docker.registry:5000/weather/poller:$(POLLER_VERSION) + +docker-build-weather: + docker build --force-rm -t docker.registry:5000/weather/server:$(WEATHER_VERSION) --build-arg WEATHER_VERSION=$(WEATHER_VERSION) -f DockerfileWeather . docker push docker.registry:5000/weather/server:$(WEATHER_VERSION) change-log-level: diff --git a/cmd/poller/main.go b/cmd/poller/main.go index d2eba8d..4a35880 100644 --- a/cmd/poller/main.go +++ b/cmd/poller/main.go @@ -1,12 +1,14 @@ package main import ( + "context" "flag" "go/weather/internal" "go/weather/internal/poller" "go/weather/internal/storage" "go/weather/internal/web" "go/weather/pkg/logger" + "io" "os" "os/signal" "strings" @@ -16,8 +18,6 @@ import ( "go.uber.org/zap/zapcore" ) -const bucket = "weather" - func main() { f := flagDefinition() @@ -31,13 +31,15 @@ func main() { addr := web.NewListenAddr("api.openweathermap.org", 443) poller := poller.NewWeatherPoller(defaultLogger, addr, 45.75, 4.85, config.OpenweatherSecret) - s3storage := storage.NewS3Storage(defaultLogger, config.S3Storage, bucket) + s3storage := storage.NewS3Storage(defaultLogger, &config.S3Storage) + var data io.ReadCloser go func() { for { - data := poller.Poll() - defer data.Close() - s3storage.Store(data) + data = poller.Poll() + if data != nil { + s3storage.Store(context.Background(), data) + } defaultLogger.Sugar().Debug("Wait next poll") time.Sleep(f.interval) @@ -48,7 +50,7 @@ func main() { signal.Notify(c, os.Interrupt) <-c - + data.Close() defaultLogger.Sugar().Info("shutting down") os.Exit(0) } diff --git a/cmd/weather/main.go b/cmd/weather/main.go index 90e8a87..f8bc050 100644 --- a/cmd/weather/main.go +++ b/cmd/weather/main.go @@ -3,6 +3,7 @@ package main import ( "context" "flag" + "go/weather/internal" "go/weather/internal/version" "go/weather/internal/web" "go/weather/pkg/logger" @@ -16,7 +17,9 @@ import ( func main() { var wait time.Duration + var configFile string flag.DurationVar(&wait, "graceful-timeout", time.Second*15, "the duration for which the server gracefully wait for existing connections to finish - e.g. 15s or 1m") + flag.StringVar(&configFile, "filename", "config.hcl", "configuration filename") flag.Parse() //logger @@ -24,13 +27,16 @@ func main() { defaultLogger := logger.NewLogger("weather", "weather.log", loggerLevel) defer defaultLogger.Sync() + //configuration parsing + config := internal.ParseConfiguration(defaultLogger.Sugar(), configFile) + //http addr := web.NewListenAddr("0.0.0.0", 8080) defaultLogger.Sugar().Infof("Weather server is listening on %s", addr) server := web.New(defaultLogger, addr, version.String()). WithTLSConfigure(). - WithHandler(). + WithHandler(config). WithHTTPLogging(). WithErrorLogging() diff --git a/config.sample.hcl b/config.sample.hcl index 6cded2c..3a0949b 100644 --- a/config.sample.hcl +++ b/config.sample.hcl @@ -1,8 +1,8 @@ openweather_secret = "" -// this block is optional and override ~/.aws/config s3 { endpoint_url = "" + bucket_name = "" region = "" aws_access_key_id = "" aws_secret_access_key = "" diff --git a/go.mod b/go.mod index c5698a7..6a83e8a 100644 --- a/go.mod +++ b/go.mod @@ -6,20 +6,33 @@ replace github.com/hashicorp/hcl/v2 => antoine-roux.tk/projects/go/hcl/v2 v2.9.0 require ( github.com/aws/aws-sdk-go v1.37.23 + github.com/dustin/go-humanize v1.0.0 // indirect github.com/elastic/go-sysinfo v1.6.0 // indirect github.com/elastic/go-windows v1.0.1 // indirect + github.com/google/uuid v1.2.0 // indirect github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-version v1.2.1 github.com/hashicorp/hcl/v2 v2.9.0 + github.com/klauspost/cpuid/v2 v2.0.6 // indirect github.com/magefile/mage v1.11.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.10 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/pelletier/go-toml v1.8.1 github.com/prometheus/procfs v0.6.0 // indirect + github.com/rs/xid v1.3.0 // indirect + github.com/sirupsen/logrus v1.6.0 // indirect go.elastic.co/apm v1.11.0 + go.elastic.co/apm/module/apmhttp v1.11.0 // indirect go.elastic.co/apm/module/apmzap v1.11.0 go.elastic.co/ecszap v1.0.0 go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.16.0 - golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46 // indirect + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c // indirect + golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect + golang.org/x/text v0.3.6 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 howett.net/plist v0.0.0-20201203080718-1454fab16a06 // indirect ) diff --git a/go.sum b/go.sum index 82a0d2a..cb715d2 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAEl github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= github.com/elastic/go-sysinfo v1.6.0 h1:u0QbU8eWSwKRPcFQancnSY4Zi0COksCJXkUgPHxE5Tw= github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= @@ -35,8 +37,16 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= @@ -49,8 +59,21 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI= +github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= @@ -62,8 +85,26 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LE github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls= github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.10 h1:1oUKe4EOPUEhw2qnPQaPsJ0lmVTYLFu03SiItauXs94= +github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -78,10 +119,20 @@ github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3x github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= +github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= @@ -106,6 +157,8 @@ github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY3 github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= go.elastic.co/apm v1.11.0 h1:uJyt6nCW9880sZhfl1tB//Jy/5TadNoAd8edRUtgb3w= go.elastic.co/apm v1.11.0/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0= +go.elastic.co/apm/module/apmhttp v1.11.0 h1:k/MjK0y2aLOXumoM8jcWXqxvIFlMS4U8Bn9cMUPdVX0= +go.elastic.co/apm/module/apmhttp v1.11.0/go.mod h1:5JFMIxdeS4vJy+D1PPPjINuX6hZ3AHalZXoOgyqZAkk= go.elastic.co/apm/module/apmzap v1.11.0 h1:KhAsnIorvxH9ar8tYV8jDM5wChaw5gnISdgInBwnomc= go.elastic.co/apm/module/apmzap v1.11.0/go.mod h1:sszgX2DXX5tF1SdxV+gYZNdol4s4JEM1qswxK0ghQ34= go.elastic.co/ecszap v1.0.0 h1:PdQkRUeraR3XHJ14T7JMa+ncU0XXrVrcEN/BoRa2nMI= @@ -134,6 +187,10 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -146,8 +203,12 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c h1:KHUzaHIpjWVlVVNh65G3hhuj3KB1HnjY6Cq5cTvRQT8= +golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -156,21 +217,35 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVs golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46 h1:V066+OYJ66oTjnhm4Yrn7SXIwSCiDQJxpBxmvqb1N1c= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210402192133-700132347e07 h1:4k6HsQjxj6hVMsI2Vf0yKlzt5lXxZsMW1q0zaq2k8zY= +golang.org/x/sys v0.0.0-20210402192133-700132347e07/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -189,6 +264,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/config.go b/internal/config.go index bdf0f62..5079242 100644 --- a/internal/config.go +++ b/internal/config.go @@ -16,8 +16,8 @@ import ( //WeatherConfig weather application relative configuration file // this configuration is common to weather server and poller type WeatherConfig struct { - OpenweatherSecret string `hcl:"openweather_secret"` - S3Storage *storage.WeatherS3StorageConfig `hcl:"s3,block"` + OpenweatherSecret string `hcl:"openweather_secret"` + S3Storage storage.WeatherS3StorageConfig `hcl:"s3,block"` } //ParseConfiguration parse configuration from filename path diff --git a/internal/poller/weather.go b/internal/poller/weather.go index 4803954..e980108 100644 --- a/internal/poller/weather.go +++ b/internal/poller/weather.go @@ -42,17 +42,14 @@ func NewWeatherPoller(log *logger.WeatherLogger, remote web.ListenAddr, lat, lon //Poll retrieve weather information func (w *weatherPoller) Poll() io.ReadCloser { r, err := w.client.Get(w.endpoint.String()) - w.logger.Info("HTTP poll to openweathermap", - zap.Int("http.response.status_code", r.StatusCode), - zap.Int("http.response.body.bytes", int(r.ContentLength)), - ) - w.handleError(err) - return r.Body -} - -//handleError generic fatal error -func (w *weatherPoller) handleError(err error) { if err != nil { w.logger.Error("Poller error", zap.Error(err)) + return nil + } else { + w.logger.Info("HTTP poll to openweathermap", + zap.Int("http.response.status_code", r.StatusCode), + zap.Int("http.response.body.bytes", int(r.ContentLength)), + ) + return r.Body } } diff --git a/internal/storage/file.go b/internal/storage/file.go index f06bab8..dc0fb62 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -1,6 +1,7 @@ package storage import ( + "context" "go/weather/pkg/logger" "io" "io/fs" @@ -25,7 +26,7 @@ func NewFileStorage(log *logger.WeatherLogger, filename string) Storage { } //Store send data to s3 bucket -func (ss *fileStorage) Store(content io.Reader) { +func (ss *fileStorage) Store(ctx context.Context, content io.Reader) { b, err := io.ReadAll(content) ss.handleError(err) diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 66ccc6e..1582e83 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -1,30 +1,24 @@ package storage import ( + "context" "fmt" "go/weather/pkg/logger" "io" - "os" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/defaults" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - toml "github.com/pelletier/go-toml" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "go.elastic.co/apm" + "go.elastic.co/apm/module/apmzap" "go.uber.org/zap" ) //AwsDefaultSection toml default section type AwsDefaultSection struct { // attribute should be public ! for go-toml - EndpointURL string `toml:"endpoint_url" hcl:"endpoint_url"` -} - -//CustomAwsConfig custom toml config for aws -type CustomAwsConfig struct { - Default AwsDefaultSection `toml:"default"` + EndpointURL string `hcl:"endpoint_url"` + BucketName string `hcl:"bucket_name"` } //WeatherS3StorageConfig interface abstracting storageConfig @@ -36,79 +30,46 @@ type WeatherS3StorageConfig struct { } //s3Storage classe used to store oi.Reader to s3 -type s3Storage struct { - logger *logger.WeatherLogger - session *session.Session - bucket string +type S3Storage struct { + logger *logger.WeatherLogger + Session *minio.Client + S3Config *WeatherS3StorageConfig } //NewS3Storage instanciate storage object -func NewS3Storage(log *logger.WeatherLogger, config *WeatherS3StorageConfig, bucket string) Storage { - s3 := s3Storage{ - logger: log, - bucket: bucket, +func NewS3Storage(log *logger.WeatherLogger, config *WeatherS3StorageConfig) *S3Storage { + s3 := S3Storage{ + logger: log, + S3Config: config, } + var err error - var consolidateConfig aws.Config - // when no config file arer specify overload weather configurable parameter with default one - if config == nil { - customConfig := CustomAwsConfig{} - err := decodeFile(defaults.SharedConfigFilename(), &customConfig) - if err != nil { - s3.logger.Fatal("Storage error", zap.Error(err)) - } - - consolidateConfig = aws.Config{ - Endpoint: &customConfig.Default.EndpointURL, - } - } else { - consolidateConfig = aws.Config{ - Endpoint: &config.EndpointURL, - Credentials: credentials.NewStaticCredentials(config.AwsAccessKeyID, config.AwsSecretAccessKey, ""), - Region: &config.Region, - } - } - - consolidateConfig.MergeIn(&aws.Config{ - DisableSSL: aws.Bool(true), - LogLevel: aws.LogLevel(log.GetAwsLevel()), - S3ForcePathStyle: aws.Bool(true), - }) - - s, err := session.NewSessionWithOptions(session.Options{ - SharedConfigState: session.SharedConfigEnable, - Config: consolidateConfig, + // Initialize minio client object. + s3.Session, err = minio.New(config.EndpointURL, &minio.Options{ + Region: config.Region, + Creds: credentials.NewStaticV4(config.AwsAccessKeyID, config.AwsSecretAccessKey, ""), + Secure: false, }) if err != nil { s3.logger.Fatal("Storage error", zap.Error(err)) } - s3.session = s return &s3 } //Store send data to s3 bucket -func (ss *s3Storage) Store(content io.Reader) { - uploader := s3manager.NewUploader(ss.session) +func (ss *S3Storage) Store(ctx context.Context, content io.Reader) { + span, ctx := apm.StartSpan(ctx, "s3StoreWeatherInfo", "custom") + defer span.End() - _, err := uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(ss.bucket), - Key: aws.String(fmt.Sprintf("%s.json", time.Now().UTC().Format(time.RFC3339))), - Body: content, + name := fmt.Sprintf("%s.json", time.Now().UTC().Format(time.RFC3339)) + + _, err := ss.Session.PutObject(ctx, ss.S3Config.BucketName, name, content, -1, minio.PutObjectOptions{ + ContentType: "application/json", }) if err != nil { ss.logger.Error("Storage error", zap.Error(err)) + } else { + ss.logger.Debug("Storage success", apmzap.TraceContext(ctx)...) } } - -//DecodeFile call toml.Decode with file -func decodeFile(fpath string, v interface{}) error { - bs, err := os.Open(fpath) - if err != nil { - return err - } - d := toml.NewDecoder(bs) - err = d.Decode(v) - - return err -} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index f00f172..60cf762 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1,8 +1,11 @@ package storage -import "io" +import ( + "context" + "io" +) //Storage capacity type Storage interface { - Store(content io.Reader) + Store(ctx context.Context, content io.Reader) } diff --git a/internal/version/version.go b/internal/version/dev.go similarity index 70% rename from internal/version/version.go rename to internal/version/dev.go index 20dce3e..9424c16 100644 --- a/internal/version/version.go +++ b/internal/version/dev.go @@ -1,24 +1,17 @@ +// +build dev + package version import ( "fmt" - - version "github.com/hashicorp/go-version" ) //Version ... -const Version = "1.0.0" +var Version = "0.1.0" //Prerelease such as "dev" (in development), "beta", "rc1", etc. var Prerelease = "dev" -//SemVer management -var SemVer *version.Version - -func init() { - SemVer = version.Must(version.NewVersion(Version)) -} - // Header is the header name used to send the current in http requests. const Header = "Weather-Version" diff --git a/internal/version/prod.go b/internal/version/prod.go new file mode 100644 index 0000000..4032115 --- /dev/null +++ b/internal/version/prod.go @@ -0,0 +1,30 @@ +// +build prod + +package version + +import ( + "fmt" + + version "github.com/hashicorp/go-version" +) + +//Version ... +var Version = "1.0.0" + +//Prerelease such as "dev" (in development), "beta", "rc1", etc. +var Prerelease = "prod" + +//SemVer management +var SemVer *version.Version + +func init() { + SemVer = version.Must(version.NewSemver(fmt.Sprintf("%s-%s", Version, Prerelease))) +} + +// Header is the header name used to send the current in http requests. +const Header = "Weather-Version" + +// String returns the complete version string, including prerelease +func String() string { + return SemVer.String() +} diff --git a/internal/web/handler.go b/internal/web/handler.go new file mode 100644 index 0000000..62b21c0 --- /dev/null +++ b/internal/web/handler.go @@ -0,0 +1,113 @@ +package web + +import ( + "encoding/json" + "fmt" + "go/weather/internal/storage" + "go/weather/pkg/headers" + "go/weather/pkg/logger" + "io" + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/minio/minio-go/v7" + "go.elastic.co/apm" + "go.elastic.co/apm/module/apmzap" + "go.uber.org/zap" +) + +const ( + welcomeMessage = "Welcome to the weather API" +) + +// WeatherHandler object grouping all handler for http web server +type WeatherHandler struct { + baseRouter *mux.Router + subRouters map[string]*mux.Router + wlogger *logger.WeatherLogger + storage *storage.S3Storage +} + +func NewWeatherHandler(logger *logger.WeatherLogger, s3 *storage.S3Storage, baseRouter *mux.Router) *WeatherHandler { + wh := WeatherHandler{ + baseRouter: baseRouter, + subRouters: make(map[string]*mux.Router, 1), + wlogger: logger, + storage: s3, + } + return &wh +} + +//getOrCreateSubRouterByName return a router by name or create if it doesn't exist yet +func (wh *WeatherHandler) GetOrCreateSubRouterByName(name, prefix string) *mux.Router { + if api := wh.subRouters[name]; api == nil { + wh.subRouters[name] = wh.baseRouter.PathPrefix(prefix).Subrouter() + } + return wh.subRouters[name] +} + +//RegisterBase base endpoint +func (wh *WeatherHandler) RegisterBase() { + wh.baseRouter.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) { + rw.Write([]byte(welcomeMessage)) + }) +} + +//RegisterApi health endpoint +func (wh *WeatherHandler) RegisterApi(version string) { + api := wh.GetOrCreateSubRouterByName(apiRouterName, apiPrefix) + + api.HandleFunc("/health", func(rw http.ResponseWriter, r *http.Request) { + rw.Header().Add(headers.ContentType, "application/json") + json.NewEncoder(rw).Encode(map[string]interface{}{ + "ok": true, + "version": version, + }) + }) + + api.HandleFunc("/all", func(rw http.ResponseWriter, r *http.Request) { + span, ctx := apm.StartSpan(r.Context(), "s3GetAllWeather", "custom") + defer span.End() + + // List all objects from a bucket-name with a matching prefix. + for object := range wh.storage.Session.ListObjects(ctx, wh.storage.S3Config.BucketName, minio.ListObjectsOptions{}) { + if object.Err != nil { + fmt.Println(object.Err) + } + fmt.Println(object) + } + }) + + api.HandleFunc("/at/{atDate}", func(rw http.ResponseWriter, r *http.Request) { + span, ctx := apm.StartSpan(r.Context(), "s3GetWeatherInfo", "custom") + defer span.End() + + vars := mux.Vars(r) + atDate, err := time.Parse(time.RFC3339, vars["atDate"]) + if err != nil { + wh.wlogger.Error(fmt.Sprintf("AtDate parse invalid format %s", vars["atDate"]), zap.Error(err)) + rw.WriteHeader(http.StatusBadRequest) + } else { + wh.wlogger.Debug("AtDate GetObject", append( + apmzap.TraceContext(ctx), + zap.String("name", fmt.Sprintf("%s.json", atDate.Format(time.RFC3339))), + )...) + reader, err := wh.storage.Session.GetObject(ctx, wh.storage.S3Config.BucketName, fmt.Sprintf("%s.json", atDate.Format(time.RFC3339)), minio.GetObjectOptions{}) + if err != nil { + wh.wlogger.Error("AtDate get s3Object failed", zap.Error(err)) + rw.WriteHeader(http.StatusInternalServerError) + } + defer reader.Close() + + rw.Header().Add(headers.ContentType, "application/json") + nb, err := io.Copy(rw, reader) + if err != nil { + wh.wlogger.Error("AtDate write response failed", zap.Error(err)) + rw.WriteHeader(http.StatusInternalServerError) + } else { + wh.wlogger.Debugf("AtDate write %d", []interface{}{nb}, apmzap.TraceContext(ctx)...) + } + } + }) +} diff --git a/internal/web/server.go b/internal/web/server.go index 5fb7f52..3d89ca1 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -3,8 +3,10 @@ package web import ( "context" "crypto/tls" - "encoding/json" "fmt" + "go/weather/internal" + "go/weather/internal/storage" + "go/weather/pkg/headers" "go/weather/pkg/logger" "net" "net/http" @@ -12,7 +14,6 @@ import ( "time" "github.com/gorilla/mux" - "go.elastic.co/apm" "go.uber.org/zap" ) @@ -24,50 +25,38 @@ const ( //WeatherServer http server for weather application type WeatherServer struct { http.Server - router *mux.Router - wlogger *logger.WeatherLogger - subRouters map[string]*mux.Router - version string + router *mux.Router + wlogger *logger.WeatherLogger + whandler *WeatherHandler + version string } //New construct new instance func New(logger *logger.WeatherLogger, addr ListenAddr, version string) *WeatherServer { + return &WeatherServer{ Server: http.Server{ Addr: addr.String(), BaseContext: func(l net.Listener) context.Context { - // here configure context properties - dt := apm.DefaultTracer - tx := dt.StartTransactionOptions("http", "request", apm.TransactionOptions{ - Start: time.Now(), - }) - defer tx.End() - return apm.ContextWithTransaction(context.Background(), tx) + return context.Background() }, WriteTimeout: 15 * time.Second, ReadTimeout: 15 * time.Second, }, - router: mux.NewRouter(), - wlogger: logger, - subRouters: make(map[string]*mux.Router, 1), - version: version, + router: mux.NewRouter(), + wlogger: logger, + version: version, } } //WithHandler register weather handler -func (ws *WeatherServer) WithHandler() *WeatherServer { +func (ws *WeatherServer) WithHandler(config *internal.WeatherConfig) *WeatherServer { + storage := storage.NewS3Storage(ws.wlogger, &config.S3Storage) + ws.whandler = NewWeatherHandler(ws.wlogger, storage, ws.router) - ws.router.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) { - rw.Write([]byte("coucou")) - }) - - api := ws.getOrCreateSubRouterByName(apiRouterName, apiPrefix) - api.HandleFunc("/health", func(rw http.ResponseWriter, r *http.Request) { - json.NewEncoder(rw).Encode(map[string]interface{}{ - "ok": true, - "version": ws.version, - }) - }) + // register each subrouter + ws.whandler.RegisterBase() + ws.whandler.RegisterApi(ws.version) return ws } @@ -76,8 +65,11 @@ func (ws *WeatherServer) WithHandler() *WeatherServer { func (ws *WeatherServer) WithHTTPLogging() *WeatherServer { ws.Server.Handler = ws.wlogger.HTTPLogHandler(ws.router) - api := ws.getOrCreateSubRouterByName(apiRouterName, apiPrefix) - api.HandleFunc("/log", ws.wlogger.Level.ServeHTTP) + api := ws.whandler.GetOrCreateSubRouterByName(apiRouterName, apiPrefix) + api.HandleFunc("/log", func(rw http.ResponseWriter, r *http.Request) { + rw.Header().Add(headers.ContentType, "application/json") + ws.wlogger.Level.ServeHTTP(rw, r) + }) return ws } @@ -116,14 +108,6 @@ func (ws *WeatherServer) Serve() error { return ws.Server.ListenAndServeTLS("", "") } -//getOrCreateSubRouterByName return a router by name or create if it doesn't exist yet -func (ws *WeatherServer) getOrCreateSubRouterByName(name, prefix string) *mux.Router { - if api := ws.subRouters[name]; api == nil { - ws.subRouters[name] = ws.router.PathPrefix(prefix).Subrouter() - } - return ws.subRouters[name] -} - //handleError generic fatal error func (ws *WeatherServer) handleError(err error) { if err != nil { diff --git a/manifests/common.tf b/manifests/common.tf index 0d9244a..f3223b0 100644 --- a/manifests/common.tf +++ b/manifests/common.tf @@ -14,6 +14,7 @@ resource "kubernetes_config_map" "weather_config" { openweather_secret = "${var.openweather_secret}" s3 { endpoint_url = "${var.S3_endpoint}" + bucket_name = "${var.S3_bucket}" region = "${var.S3_region}" aws_access_key_id = "${var.S3_key_id}" aws_secret_access_key = "${var.S3_key_secret}" @@ -38,8 +39,9 @@ module "weather_server_application" { environment = var.environment application_name = "weather-server" kubernetes_namespace = kubernetes_namespace.application_namespace - application_image = format("docker.registry/weather/server:%s", var.poller_version) + application_image = format("docker.registry/weather/server:%s", var.weather_version) kubernetes_config_map = kubernetes_config_map.weather_config.metadata.0 expose_application = true application_dns = "weather.localdomain" + application_args = ["-filename", "/conf/config.hcl"] } \ No newline at end of file diff --git a/manifests/configure.tf b/manifests/configure.tf index 10f1995..415de46 100644 --- a/manifests/configure.tf +++ b/manifests/configure.tf @@ -9,7 +9,7 @@ terraform { } backend "etcdv3" { - endpoints = ["https://dx30.localdomain:2379"] + endpoints = ["https://100.64.0.19:2379"] lock = true prefix = "/terraform-state/weather/" diff --git a/manifests/variables.tf b/manifests/variables.tf index 781b5c2..5834025 100644 --- a/manifests/variables.tf +++ b/manifests/variables.tf @@ -14,6 +14,10 @@ variable "S3_endpoint" { default = "s3.localdomain" description = "S3 backend endpoint" } +variable "S3_bucket" { + default = "weather" + description = "S3 bucket" +} variable "S3_region" { default = "FR" description = "S3 backend region" @@ -24,10 +28,10 @@ variable "S3_key_id" { } variable "poller_version" { - default = "latest" + default = "1.0.0" description = "poller container version" } variable "weather_version" { - default = "latest" + default = "1.0.0" description = "poller container version" } \ No newline at end of file diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index aee7487..0c5b069 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -2,6 +2,7 @@ package logger import ( "bytes" + "fmt" "go/weather/pkg/headers" "io" "io/ioutil" @@ -9,8 +10,10 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/aws/aws-sdk-go/aws" + "go.elastic.co/apm" "go.elastic.co/apm/module/apmzap" "go.elastic.co/ecszap" "go.uber.org/zap" @@ -81,15 +84,18 @@ func NewLogger(loggerName, logPath string, level zap.AtomicLevel) *WeatherLogger //HTTPLogHandler http handler to log http request and http response func (wl *WeatherLogger) HTTPLogHandler(next http.Handler) http.Handler { - // dt := apm.DefaultTracer return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // here configure context properties + tx := apm.DefaultTracer.StartTransactionOptions("http", "request", apm.TransactionOptions{ + Start: time.Now(), + }) + defer tx.End() + ctxtt := apm.ContextWithTransaction(r.Context(), tx) + r = r.WithContext(ctxtt) + ww := ResponseWriter{ ResponseWriter: w, } - - // span, ctx := apm.StartSpan(r.Context(), "updateRequestCount", "custom") - // defer span.End() - wl.LogHTTPRequest(r) next.ServeHTTP(&ww, r) wl.LogHTTPResponse(ww) @@ -163,3 +169,13 @@ func (wl *WeatherLogger) GetAwsLevel() aws.LogLevelType { return aws.LogOff } } + +//Debugf Debug + fmt.Sprintf +func (wl *WeatherLogger) Debugf(msg string, interpolate []interface{}, field ...zapcore.Field) { + wl.Logger.Debug(fmt.Sprintf(msg, interpolate), field...) +} + +//Debugf Error + fmt.Sprintf +func (wl *WeatherLogger) Errorf(msg string, interpolate []interface{}, field ...zapcore.Field) { + wl.Logger.Error(fmt.Sprintf(msg, interpolate), field...) +}