Browse Source

override version at build time, fix trace id add span id, configure GET /at/date handler

master
RouxAntoine 1 week ago
parent
commit
55aeb4867a
Signed by: antoine <antoinroux@hotmail.fr> GPG Key ID: 098FB66FC0475E70
22 changed files with 378 additions and 170 deletions
  1. +2
    -1
      .dockerignore
  2. +3
    -2
      DockerfilePoller
  3. +3
    -2
      DockerfileWeather
  4. +14
    -13
      Makefile
  5. +9
    -7
      cmd/poller/main.go
  6. +7
    -1
      cmd/weather/main.go
  7. +1
    -1
      config.sample.hcl
  8. +14
    -1
      go.mod
  9. +79
    -0
      go.sum
  10. +2
    -2
      internal/config.go
  11. +7
    -10
      internal/poller/weather.go
  12. +2
    -1
      internal/storage/file.go
  13. +30
    -69
      internal/storage/s3.go
  14. +5
    -2
      internal/storage/storage.go
  15. +3
    -10
      internal/version/dev.go
  16. +30
    -0
      internal/version/prod.go
  17. +113
    -0
      internal/web/handler.go
  18. +23
    -39
      internal/web/server.go
  19. +3
    -1
      manifests/common.tf
  20. +1
    -1
      manifests/configure.tf
  21. +6
    -2
      manifests/variables.tf
  22. +21
    -5
      pkg/logger/logger.go

+ 2
- 1
.dockerignore View File

@@ -7,4 +7,5 @@ db.json
*.log
config.hcl
Dockerfile
.gitignore
.gitignore
manifests/

+ 3
- 2
DockerfilePoller View File

@@ -32,12 +32,13 @@ COPY go.mod .
COPY go.sum .
RUN make dependencies

ARG POLLER_VERSION
COPY . .
RUN make build-poller \
GOARCH=$TARGETARCH \
GOOS=$(echo $BUILDPLATFORM | cut -d'/' -f1) \
GOBUILDFLAGS="-a -tags netgo -installsuffix netgo" \
LDFLAGS="-w -s -d"
GOBUILDFLAGS="-a -tags netgo,prod -installsuffix netgo" \
LDFLAGS="-w -s -d -X go/weather/internal/version.Version=$POLLER_VERSION"

FROM scratch



+ 3
- 2
DockerfileWeather View File

@@ -35,13 +35,14 @@ COPY go.mod .
COPY go.sum .
RUN make dependencies

ARG WEATHER_VERSION
COPY . .
# RUN make build-$BINARYNAME GOARCH=$TARGETARCH GOOS=$(echo $BUILDPLATFORM | cut -d'/' -f1) LDFLAGS='-extldflags="-static"'
RUN make build-weather \
GOARCH=$TARGETARCH \
GOOS=$(echo $BUILDPLATFORM | cut -d'/' -f1) \
GOBUILDFLAGS="-a -tags netgo -installsuffix netgo" \
LDFLAGS="-w -s -d"
GOBUILDFLAGS=" -a -tags netgo,prod -installsuffix netgo" \
LDFLAGS="-w -s -d -X go/weather/internal/version.Version=$WEATHER_VERSION"

FROM alpine



+ 14
- 13
Makefile View File

@@ -5,19 +5,21 @@ GOARCH=amd64
# GOARCH=arm
GOOS=darwin
# GOOS=linux
LDFLAGS=-w -s
GOBUILDFLAGS=
DOCKER_BUILDKIT=1
LDFLAGS=-w -s -X go/weather/internal/version.Version=$$(git rev-list -1 HEAD)
GOBUILDFLAGS=-tags dev
CGO_ENABLED=0
WEATHER_VERSION=latest
POLLER_VERSION=latest
DOMAIN_ALIAS=weather.localdomain

DOCKER_BUILDKIT=1

WEATHER_VERSION=1.0.0
POLLER_VERSION=1.0.0
DOMAIN_ALIAS=weather.localdomain,weather-dev.localdomain

build: build-poller build-weather

build-weather: dependencies
@echo "build for os $$GOOS and arch $$GOARCH"
go build -o bin/weather-$(GOOS)-$(GOARCH) cmd/weather/main.go
go build -o bin/weather-$(GOOS)-$(GOARCH) -ldflags="$(LDFLAGS)" $(GOBUILDFLAGS) cmd/weather/main.go

build-poller: dependencies
@echo "build for os $$GOOS and arch $$GOARCH"
@@ -50,15 +52,14 @@ clean:
get-root-ca:
cfssl info -remote rasp1.localdomain:444 -config certs/client-config.json | cfssljson -bare -stdout /dev/stdout | tee certs/out/ca.pem

docker: docker-build-poller docker-build-weather docker-push
docker: docker-build-poller docker-build-weather

docker-build-poller:
docker build --force-rm -t docker.registry:5000/weather/poller:$(POLLER_VERSION) -f DockerfilePoller .
docker-build-weather:
docker build --force-rm -t docker.registry:5000/weather/server:$(WEATHER_VERSION) -f DockerfileWeather .

docker-push:
docker build --force-rm -t docker.registry:5000/weather/poller:$(POLLER_VERSION) --build-arg POLLER_VERSION=$(POLLER_VERSION) -f DockerfilePoller .
docker push docker.registry:5000/weather/poller:$(POLLER_VERSION)

docker-build-weather:
docker build --force-rm -t docker.registry:5000/weather/server:$(WEATHER_VERSION) --build-arg WEATHER_VERSION=$(WEATHER_VERSION) -f DockerfileWeather .
docker push docker.registry:5000/weather/server:$(WEATHER_VERSION)

change-log-level:


+ 9
- 7
cmd/poller/main.go View File

@@ -1,12 +1,14 @@
package main

import (
"context"
"flag"
"go/weather/internal"
"go/weather/internal/poller"
"go/weather/internal/storage"
"go/weather/internal/web"
"go/weather/pkg/logger"
"io"
"os"
"os/signal"
"strings"
@@ -16,8 +18,6 @@ import (
"go.uber.org/zap/zapcore"
)

const bucket = "weather"

func main() {
f := flagDefinition()

@@ -31,13 +31,15 @@ func main() {
addr := web.NewListenAddr("api.openweathermap.org", 443)

poller := poller.NewWeatherPoller(defaultLogger, addr, 45.75, 4.85, config.OpenweatherSecret)
s3storage := storage.NewS3Storage(defaultLogger, config.S3Storage, bucket)
s3storage := storage.NewS3Storage(defaultLogger, &config.S3Storage)

var data io.ReadCloser
go func() {
for {
data := poller.Poll()
defer data.Close()
s3storage.Store(data)
data = poller.Poll()
if data != nil {
s3storage.Store(context.Background(), data)
}

defaultLogger.Sugar().Debug("Wait next poll")
time.Sleep(f.interval)
@@ -48,7 +50,7 @@ func main() {
signal.Notify(c, os.Interrupt)

<-c
data.Close()
defaultLogger.Sugar().Info("shutting down")
os.Exit(0)
}


+ 7
- 1
cmd/weather/main.go View File

@@ -3,6 +3,7 @@ package main
import (
"context"
"flag"
"go/weather/internal"
"go/weather/internal/version"
"go/weather/internal/web"
"go/weather/pkg/logger"
@@ -16,7 +17,9 @@ import (

func main() {
var wait time.Duration
var configFile string
flag.DurationVar(&wait, "graceful-timeout", time.Second*15, "the duration for which the server gracefully wait for existing connections to finish - e.g. 15s or 1m")
flag.StringVar(&configFile, "filename", "config.hcl", "configuration filename")
flag.Parse()

//logger
@@ -24,13 +27,16 @@ func main() {
defaultLogger := logger.NewLogger("weather", "weather.log", loggerLevel)
defer defaultLogger.Sync()

//configuration parsing
config := internal.ParseConfiguration(defaultLogger.Sugar(), configFile)

//http
addr := web.NewListenAddr("0.0.0.0", 8080)

defaultLogger.Sugar().Infof("Weather server is listening on %s", addr)
server := web.New(defaultLogger, addr, version.String()).
WithTLSConfigure().
WithHandler().
WithHandler(config).
WithHTTPLogging().
WithErrorLogging()



+ 1
- 1
config.sample.hcl View File

@@ -1,8 +1,8 @@
openweather_secret = ""

// this block is optional and override ~/.aws/config
s3 {
endpoint_url = ""
bucket_name = ""
region = ""
aws_access_key_id = ""
aws_secret_access_key = ""

+ 14
- 1
go.mod View File

@@ -6,20 +6,33 @@ replace github.com/hashicorp/hcl/v2 => antoine-roux.tk/projects/go/hcl/v2 v2.9.0

require (
github.com/aws/aws-sdk-go v1.37.23
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/elastic/go-sysinfo v1.6.0 // indirect
github.com/elastic/go-windows v1.0.1 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-version v1.2.1
github.com/hashicorp/hcl/v2 v2.9.0
github.com/klauspost/cpuid/v2 v2.0.6 // indirect
github.com/magefile/mage v1.11.0 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.10 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/pelletier/go-toml v1.8.1
github.com/prometheus/procfs v0.6.0 // indirect
github.com/rs/xid v1.3.0 // indirect
github.com/sirupsen/logrus v1.6.0 // indirect
go.elastic.co/apm v1.11.0
go.elastic.co/apm/module/apmhttp v1.11.0 // indirect
go.elastic.co/apm/module/apmzap v1.11.0
go.elastic.co/ecszap v1.0.0
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.16.0
golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46 // indirect
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c // indirect
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
golang.org/x/text v0.3.6 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0
howett.net/plist v0.0.0-20201203080718-1454fab16a06 // indirect
)

+ 79
- 0
go.sum View File

@@ -19,6 +19,8 @@ github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAEl
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
github.com/elastic/go-sysinfo v1.6.0 h1:u0QbU8eWSwKRPcFQancnSY4Zi0COksCJXkUgPHxE5Tw=
github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
@@ -35,8 +37,16 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
@@ -49,8 +59,21 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI=
github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
@@ -62,8 +85,26 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LE
github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls=
github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.10 h1:1oUKe4EOPUEhw2qnPQaPsJ0lmVTYLFu03SiItauXs94=
github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -78,10 +119,20 @@ github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3x
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis=
github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc=
github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
@@ -106,6 +157,8 @@ github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY3
github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
go.elastic.co/apm v1.11.0 h1:uJyt6nCW9880sZhfl1tB//Jy/5TadNoAd8edRUtgb3w=
go.elastic.co/apm v1.11.0/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0=
go.elastic.co/apm/module/apmhttp v1.11.0 h1:k/MjK0y2aLOXumoM8jcWXqxvIFlMS4U8Bn9cMUPdVX0=
go.elastic.co/apm/module/apmhttp v1.11.0/go.mod h1:5JFMIxdeS4vJy+D1PPPjINuX6hZ3AHalZXoOgyqZAkk=
go.elastic.co/apm/module/apmzap v1.11.0 h1:KhAsnIorvxH9ar8tYV8jDM5wChaw5gnISdgInBwnomc=
go.elastic.co/apm/module/apmzap v1.11.0/go.mod h1:sszgX2DXX5tF1SdxV+gYZNdol4s4JEM1qswxK0ghQ34=
go.elastic.co/ecszap v1.0.0 h1:PdQkRUeraR3XHJ14T7JMa+ncU0XXrVrcEN/BoRa2nMI=
@@ -134,6 +187,10 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -146,8 +203,12 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c h1:KHUzaHIpjWVlVVNh65G3hhuj3KB1HnjY6Cq5cTvRQT8=
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -156,21 +217,35 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVs
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46 h1:V066+OYJ66oTjnhm4Yrn7SXIwSCiDQJxpBxmvqb1N1c=
golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210402192133-700132347e07 h1:4k6HsQjxj6hVMsI2Vf0yKlzt5lXxZsMW1q0zaq2k8zY=
golang.org/x/sys v0.0.0-20210402192133-700132347e07/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A=
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -189,6 +264,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=


+ 2
- 2
internal/config.go View File

@@ -16,8 +16,8 @@ import (
//WeatherConfig weather application relative configuration file
// this configuration is common to weather server and poller
type WeatherConfig struct {
OpenweatherSecret string `hcl:"openweather_secret"`
S3Storage *storage.WeatherS3StorageConfig `hcl:"s3,block"`
OpenweatherSecret string `hcl:"openweather_secret"`
S3Storage storage.WeatherS3StorageConfig `hcl:"s3,block"`
}

//ParseConfiguration parse configuration from filename path


+ 7
- 10
internal/poller/weather.go View File

@@ -42,17 +42,14 @@ func NewWeatherPoller(log *logger.WeatherLogger, remote web.ListenAddr, lat, lon
//Poll retrieve weather information
func (w *weatherPoller) Poll() io.ReadCloser {
r, err := w.client.Get(w.endpoint.String())
w.logger.Info("HTTP poll to openweathermap",
zap.Int("http.response.status_code", r.StatusCode),
zap.Int("http.response.body.bytes", int(r.ContentLength)),
)
w.handleError(err)
return r.Body
}

//handleError generic fatal error
func (w *weatherPoller) handleError(err error) {
if err != nil {
w.logger.Error("Poller error", zap.Error(err))
return nil
} else {
w.logger.Info("HTTP poll to openweathermap",
zap.Int("http.response.status_code", r.StatusCode),
zap.Int("http.response.body.bytes", int(r.ContentLength)),
)
return r.Body
}
}

+ 2
- 1
internal/storage/file.go View File

@@ -1,6 +1,7 @@
package storage

import (
"context"
"go/weather/pkg/logger"
"io"
"io/fs"
@@ -25,7 +26,7 @@ func NewFileStorage(log *logger.WeatherLogger, filename string) Storage {
}

//Store send data to s3 bucket
func (ss *fileStorage) Store(content io.Reader) {
func (ss *fileStorage) Store(ctx context.Context, content io.Reader) {
b, err := io.ReadAll(content)
ss.handleError(err)



+ 30
- 69
internal/storage/s3.go View File

@@ -1,30 +1,24 @@
package storage

import (
"context"
"fmt"
"go/weather/pkg/logger"
"io"
"os"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
toml "github.com/pelletier/go-toml"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"go.elastic.co/apm"
"go.elastic.co/apm/module/apmzap"
"go.uber.org/zap"
)

//AwsDefaultSection toml default section
type AwsDefaultSection struct {
// attribute should be public ! for go-toml
EndpointURL string `toml:"endpoint_url" hcl:"endpoint_url"`
}

//CustomAwsConfig custom toml config for aws
type CustomAwsConfig struct {
Default AwsDefaultSection `toml:"default"`
EndpointURL string `hcl:"endpoint_url"`
BucketName string `hcl:"bucket_name"`
}

//WeatherS3StorageConfig interface abstracting storageConfig
@@ -36,79 +30,46 @@ type WeatherS3StorageConfig struct {
}

//s3Storage classe used to store oi.Reader to s3
type s3Storage struct {
logger *logger.WeatherLogger
session *session.Session
bucket string
type S3Storage struct {
logger *logger.WeatherLogger
Session *minio.Client
S3Config *WeatherS3StorageConfig
}

//NewS3Storage instanciate storage object
func NewS3Storage(log *logger.WeatherLogger, config *WeatherS3StorageConfig, bucket string) Storage {
s3 := s3Storage{
logger: log,
bucket: bucket,
}

var consolidateConfig aws.Config
// when no config file arer specify overload weather configurable parameter with default one
if config == nil {
customConfig := CustomAwsConfig{}
err := decodeFile(defaults.SharedConfigFilename(), &customConfig)
if err != nil {
s3.logger.Fatal("Storage error", zap.Error(err))
}

consolidateConfig = aws.Config{
Endpoint: &customConfig.Default.EndpointURL,
}
} else {
consolidateConfig = aws.Config{
Endpoint: &config.EndpointURL,
Credentials: credentials.NewStaticCredentials(config.AwsAccessKeyID, config.AwsSecretAccessKey, ""),
Region: &config.Region,
}
func NewS3Storage(log *logger.WeatherLogger, config *WeatherS3StorageConfig) *S3Storage {
s3 := S3Storage{
logger: log,
S3Config: config,
}
var err error

consolidateConfig.MergeIn(&aws.Config{
DisableSSL: aws.Bool(true),
LogLevel: aws.LogLevel(log.GetAwsLevel()),
S3ForcePathStyle: aws.Bool(true),
})

s, err := session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Config: consolidateConfig,
// Initialize minio client object.
s3.Session, err = minio.New(config.EndpointURL, &minio.Options{
Region: config.Region,
Creds: credentials.NewStaticV4(config.AwsAccessKeyID, config.AwsSecretAccessKey, ""),
Secure: false,
})
if err != nil {
s3.logger.Fatal("Storage error", zap.Error(err))
}
s3.session = s

return &s3
}

//Store send data to s3 bucket
func (ss *s3Storage) Store(content io.Reader) {
uploader := s3manager.NewUploader(ss.session)
func (ss *S3Storage) Store(ctx context.Context, content io.Reader) {
span, ctx := apm.StartSpan(ctx, "s3StoreWeatherInfo", "custom")
defer span.End()

_, err := uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(ss.bucket),
Key: aws.String(fmt.Sprintf("%s.json", time.Now().UTC().Format(time.RFC3339))),
Body: content,
name := fmt.Sprintf("%s.json", time.Now().UTC().Format(time.RFC3339))
_, err := ss.Session.PutObject(ctx, ss.S3Config.BucketName, name, content, -1, minio.PutObjectOptions{
ContentType: "application/json",
})
if err != nil {
ss.logger.Error("Storage error", zap.Error(err))
} else {
ss.logger.Debug("Storage success", apmzap.TraceContext(ctx)...)
}
}

//DecodeFile call toml.Decode with file
func decodeFile(fpath string, v interface{}) error {
bs, err := os.Open(fpath)
if err != nil {
return err
}
d := toml.NewDecoder(bs)
err = d.Decode(v)

return err
}

+ 5
- 2
internal/storage/storage.go View File

@@ -1,8 +1,11 @@
package storage

import "io"
import (
"context"
"io"
)

//Storage capacity
type Storage interface {
Store(content io.Reader)
Store(ctx context.Context, content io.Reader)
}

internal/version/version.go → internal/version/dev.go View File

@@ -1,24 +1,17 @@
// +build dev

package version

import (
"fmt"

version "github.com/hashicorp/go-version"
)

//Version ...
const Version = "1.0.0"
var Version = "0.1.0"

//Prerelease such as "dev" (in development), "beta", "rc1", etc.
var Prerelease = "dev"

//SemVer management
var SemVer *version.Version

func init() {
SemVer = version.Must(version.NewVersion(Version))
}

// Header is the header name used to send the current in http requests.
const Header = "Weather-Version"


+ 30
- 0
internal/version/prod.go View File

@@ -0,0 +1,30 @@
// +build prod

package version

import (
"fmt"

version "github.com/hashicorp/go-version"
)

//Version ...
var Version = "1.0.0"

//Prerelease such as "dev" (in development), "beta", "rc1", etc.
var Prerelease = "prod"

//SemVer management
var SemVer *version.Version

func init() {
SemVer = version.Must(version.NewSemver(fmt.Sprintf("%s-%s", Version, Prerelease)))
}

// Header is the header name used to send the current in http requests.
const Header = "Weather-Version"

// String returns the complete version string, including prerelease
func String() string {
return SemVer.String()
}

+ 113
- 0
internal/web/handler.go View File

@@ -0,0 +1,113 @@
package web

import (
"encoding/json"
"fmt"
"go/weather/internal/storage"
"go/weather/pkg/headers"
"go/weather/pkg/logger"
"io"
"net/http"
"time"

"github.com/gorilla/mux"
"github.com/minio/minio-go/v7"
"go.elastic.co/apm"
"go.elastic.co/apm/module/apmzap"
"go.uber.org/zap"
)

const (
welcomeMessage = "Welcome to the weather API"
)

// WeatherHandler object grouping all handler for http web server
type WeatherHandler struct {
baseRouter *mux.Router
subRouters map[string]*mux.Router
wlogger *logger.WeatherLogger
storage *storage.S3Storage
}

func NewWeatherHandler(logger *logger.WeatherLogger, s3 *storage.S3Storage, baseRouter *mux.Router) *WeatherHandler {
wh := WeatherHandler{
baseRouter: baseRouter,
subRouters: make(map[string]*mux.Router, 1),
wlogger: logger,
storage: s3,
}
return &wh
}

//getOrCreateSubRouterByName return a router by name or create if it doesn't exist yet
func (wh *WeatherHandler) GetOrCreateSubRouterByName(name, prefix string) *mux.Router {
if api := wh.subRouters[name]; api == nil {
wh.subRouters[name] = wh.baseRouter.PathPrefix(prefix).Subrouter()
}
return wh.subRouters[name]
}

//RegisterBase base endpoint
func (wh *WeatherHandler) RegisterBase() {
wh.baseRouter.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
rw.Write([]byte(welcomeMessage))
})
}

//RegisterApi health endpoint
func (wh *WeatherHandler) RegisterApi(version string) {
api := wh.GetOrCreateSubRouterByName(apiRouterName, apiPrefix)

api.HandleFunc("/health", func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Add(headers.ContentType, "application/json")
json.NewEncoder(rw).Encode(map[string]interface{}{
"ok": true,
"version": version,
})
})

api.HandleFunc("/all", func(rw http.ResponseWriter, r *http.Request) {
span, ctx := apm.StartSpan(r.Context(), "s3GetAllWeather", "custom")
defer span.End()

// List all objects from a bucket-name with a matching prefix.
for object := range wh.storage.Session.ListObjects(ctx, wh.storage.S3Config.BucketName, minio.ListObjectsOptions{}) {
if object.Err != nil {
fmt.Println(object.Err)
}
fmt.Println(object)
}
})

api.HandleFunc("/at/{atDate}", func(rw http.ResponseWriter, r *http.Request) {
span, ctx := apm.StartSpan(r.Context(), "s3GetWeatherInfo", "custom")
defer span.End()

vars := mux.Vars(r)
atDate, err := time.Parse(time.RFC3339, vars["atDate"])
if err != nil {
wh.wlogger.Error(fmt.Sprintf("AtDate parse invalid format %s", vars["atDate"]), zap.Error(err))
rw.WriteHeader(http.StatusBadRequest)
} else {
wh.wlogger.Debug("AtDate GetObject", append(
apmzap.TraceContext(ctx),
zap.String("name", fmt.Sprintf("%s.json", atDate.Format(time.RFC3339))),
)...)
reader, err := wh.storage.Session.GetObject(ctx, wh.storage.S3Config.BucketName, fmt.Sprintf("%s.json", atDate.Format(time.RFC3339)), minio.GetObjectOptions{})
if err != nil {
wh.wlogger.Error("AtDate get s3Object failed", zap.Error(err))
rw.WriteHeader(http.StatusInternalServerError)
}
defer reader.Close()

rw.Header().Add(headers.ContentType, "application/json")
nb, err := io.Copy(rw, reader)
if err != nil {
wh.wlogger.Error("AtDate write response failed", zap.Error(err))
rw.WriteHeader(http.StatusInternalServerError)
} else {
wh.wlogger.Debugf("AtDate write %d", []interface{}{nb}, apmzap.TraceContext(ctx)...)
}
}
})
}

+ 23
- 39
internal/web/server.go View File

@@ -3,8 +3,10 @@ package web
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"go/weather/internal"
"go/weather/internal/storage"
"go/weather/pkg/headers"
"go/weather/pkg/logger"
"net"
"net/http"
@@ -12,7 +14,6 @@ import (
"time"

"github.com/gorilla/mux"
"go.elastic.co/apm"
"go.uber.org/zap"
)

@@ -24,50 +25,38 @@ const (
//WeatherServer http server for weather application
type WeatherServer struct {
http.Server
router *mux.Router
wlogger *logger.WeatherLogger
subRouters map[string]*mux.Router
version string
router *mux.Router
wlogger *logger.WeatherLogger
whandler *WeatherHandler
version string
}

//New construct new instance
func New(logger *logger.WeatherLogger, addr ListenAddr, version string) *WeatherServer {

return &WeatherServer{
Server: http.Server{
Addr: addr.String(),
BaseContext: func(l net.Listener) context.Context {
// here configure context properties
dt := apm.DefaultTracer
tx := dt.StartTransactionOptions("http", "request", apm.TransactionOptions{
Start: time.Now(),
})
defer tx.End()
return apm.ContextWithTransaction(context.Background(), tx)
return context.Background()
},
WriteTimeout: 15 * time.Second,
ReadTimeout: 15 * time.Second,
},
router: mux.NewRouter(),
wlogger: logger,
subRouters: make(map[string]*mux.Router, 1),
version: version,
router: mux.NewRouter(),
wlogger: logger,
version: version,
}
}

//WithHandler register weather handler
func (ws *WeatherServer) WithHandler() *WeatherServer {

ws.router.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
rw.Write([]byte("coucou"))
})
func (ws *WeatherServer) WithHandler(config *internal.WeatherConfig) *WeatherServer {
storage := storage.NewS3Storage(ws.wlogger, &config.S3Storage)
ws.whandler = NewWeatherHandler(ws.wlogger, storage, ws.router)

api := ws.getOrCreateSubRouterByName(apiRouterName, apiPrefix)
api.HandleFunc("/health", func(rw http.ResponseWriter, r *http.Request) {
json.NewEncoder(rw).Encode(map[string]interface{}{
"ok": true,
"version": ws.version,
})
})
// register each subrouter
ws.whandler.RegisterBase()
ws.whandler.RegisterApi(ws.version)

return ws
}
@@ -76,8 +65,11 @@ func (ws *WeatherServer) WithHandler() *WeatherServer {
func (ws *WeatherServer) WithHTTPLogging() *WeatherServer {
ws.Server.Handler = ws.wlogger.HTTPLogHandler(ws.router)

api := ws.getOrCreateSubRouterByName(apiRouterName, apiPrefix)
api.HandleFunc("/log", ws.wlogger.Level.ServeHTTP)
api := ws.whandler.GetOrCreateSubRouterByName(apiRouterName, apiPrefix)
api.HandleFunc("/log", func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Add(headers.ContentType, "application/json")
ws.wlogger.Level.ServeHTTP(rw, r)
})
return ws
}

@@ -116,14 +108,6 @@ func (ws *WeatherServer) Serve() error {
return ws.Server.ListenAndServeTLS("", "")
}

//getOrCreateSubRouterByName return a router by name or create if it doesn't exist yet
func (ws *WeatherServer) getOrCreateSubRouterByName(name, prefix string) *mux.Router {
if api := ws.subRouters[name]; api == nil {
ws.subRouters[name] = ws.router.PathPrefix(prefix).Subrouter()
}
return ws.subRouters[name]
}

//handleError generic fatal error
func (ws *WeatherServer) handleError(err error) {
if err != nil {


+ 3
- 1
manifests/common.tf View File

@@ -14,6 +14,7 @@ resource "kubernetes_config_map" "weather_config" {
openweather_secret = "${var.openweather_secret}"
s3 {
endpoint_url = "${var.S3_endpoint}"
bucket_name = "${var.S3_bucket}"
region = "${var.S3_region}"
aws_access_key_id = "${var.S3_key_id}"
aws_secret_access_key = "${var.S3_key_secret}"
@@ -38,8 +39,9 @@ module "weather_server_application" {
environment = var.environment
application_name = "weather-server"
kubernetes_namespace = kubernetes_namespace.application_namespace
application_image = format("docker.registry/weather/server:%s", var.poller_version)
application_image = format("docker.registry/weather/server:%s", var.weather_version)
kubernetes_config_map = kubernetes_config_map.weather_config.metadata.0
expose_application = true
application_dns = "weather.localdomain"
application_args = ["-filename", "/conf/config.hcl"]
}

+ 1
- 1
manifests/configure.tf View File

@@ -9,7 +9,7 @@ terraform {
}

backend "etcdv3" {
endpoints = ["https://dx30.localdomain:2379"]
endpoints = ["https://100.64.0.19:2379"]
lock = true
prefix = "/terraform-state/weather/"


+ 6
- 2
manifests/variables.tf View File

@@ -14,6 +14,10 @@ variable "S3_endpoint" {
default = "s3.localdomain"
description = "S3 backend endpoint"
}
variable "S3_bucket" {
default = "weather"
description = "S3 bucket"
}
variable "S3_region" {
default = "FR"
description = "S3 backend region"
@@ -24,10 +28,10 @@ variable "S3_key_id" {
}

variable "poller_version" {
default = "latest"
default = "1.0.0"
description = "poller container version"
}
variable "weather_version" {
default = "latest"
default = "1.0.0"
description = "poller container version"
}

+ 21
- 5
pkg/logger/logger.go View File

@@ -2,6 +2,7 @@ package logger

import (
"bytes"
"fmt"
"go/weather/pkg/headers"
"io"
"io/ioutil"
@@ -9,8 +10,10 @@ import (
"os"
"os/signal"
"syscall"
"time"

"github.com/aws/aws-sdk-go/aws"
"go.elastic.co/apm"
"go.elastic.co/apm/module/apmzap"
"go.elastic.co/ecszap"
"go.uber.org/zap"
@@ -81,15 +84,18 @@ func NewLogger(loggerName, logPath string, level zap.AtomicLevel) *WeatherLogger

//HTTPLogHandler http handler to log http request and http response
func (wl *WeatherLogger) HTTPLogHandler(next http.Handler) http.Handler {
// dt := apm.DefaultTracer
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// here configure context properties
tx := apm.DefaultTracer.StartTransactionOptions("http", "request", apm.TransactionOptions{
Start: time.Now(),
})
defer tx.End()
ctxtt := apm.ContextWithTransaction(r.Context(), tx)
r = r.WithContext(ctxtt)

ww := ResponseWriter{
ResponseWriter: w,
}

// span, ctx := apm.StartSpan(r.Context(), "updateRequestCount", "custom")
// defer span.End()

wl.LogHTTPRequest(r)
next.ServeHTTP(&ww, r)
wl.LogHTTPResponse(ww)
@@ -163,3 +169,13 @@ func (wl *WeatherLogger) GetAwsLevel() aws.LogLevelType {
return aws.LogOff
}
}

//Debugf Debug + fmt.Sprintf
func (wl *WeatherLogger) Debugf(msg string, interpolate []interface{}, field ...zapcore.Field) {
wl.Logger.Debug(fmt.Sprintf(msg, interpolate), field...)
}

//Debugf Error + fmt.Sprintf
func (wl *WeatherLogger) Errorf(msg string, interpolate []interface{}, field ...zapcore.Field) {
wl.Logger.Error(fmt.Sprintf(msg, interpolate), field...)
}

Loading…
Cancel
Save