feat: poller working and implement s3 and file storage

.gitignore

@@ -7,4 +7,7 @@ bin/
 certs/out/
 certs/pki_auth.key
 !certs/out/.gitkeep
-*.log
+*.log
+
+db.json
+config.hcl

.vscode/launch.json

@@ -12,5 +12,13 @@
             "program": "${workspaceFolder}/cmd/weather/main.go",
             "cwd": "${workspaceFolder}"
         }
+        {
+            "name": "Run poller applicaton",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/poller/main.go",
+            "cwd": "${workspaceFolder}"
+        }
     ]
 }

Makefile

@@ -1,13 +1,17 @@
-.PHONY: gen-cert
+.PHONY: gen-cert build-weather build-poller gen-cert dev-dependencies cert-info cert-info-remote curl-test clean get-root-ca
 
+build: build-poller build-weather
 
-build:
+build-weather:
 	go build -o bin/weather cmd/weather/main.go
 
+build-poller:
+	go build -o bin/poller cmd/poller/main.go
+
 gen-cert:
 	cfssl gencert -config certs/client-config.json -profile server -hostname weather.localdomain certs/client-csr.json | cfssljson -bare certs/out/weather
 
-dependencies:
+dev-dependencies:
 	go get github.com/cloudflare/cfssl/cmd/cfssl
 	go get github.com/cloudflare/cfssl/cmd/cfssljson
 	go get github.com/cloudflare/cfssl/cmd/mkbundle

cmd/poller/main.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/weather/internal"
+	"go/weather/internal/poller"
+	"go/weather/internal/storage"
+	"go/weather/internal/web"
+	"go/weather/pkg/logger"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/gohcl"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+
+	"go.uber.org/zap"
+)
+
+const bucket = "weather"
+
+func main() {
+	var interval time.Duration
+	var filename string
+	flag.DurationVar(&interval, "check-interval", time.Second*30, "the sleep duration between multiple call to remote weather api")
+	flag.StringVar(&filename, "filename", "config.hcl", "configuration filename")
+	flag.Parse()
+
+	loggerLevel := zap.NewAtomicLevelAt(zap.DebugLevel)
+	defaultLogger := logger.NewLogger("poller", "weather.log", loggerLevel)
+	defaultLoggerSugar := defaultLogger.Sugar()
+	defer defaultLogger.Sync()
+
+	if filename == "config.hcl" {
+		dir, err := os.Getwd()
+		if err != nil {
+			defaultLoggerSugar.Fatal("Fail to determine current directory to load default ./config.hcl file")
+		}
+		filename = fmt.Sprintf("%s%c%s", dir, os.PathSeparator, filename)
+	}
+
+	src, err := ioutil.ReadFile(filename)
+	if err != nil {
+		defaultLoggerSugar.Fatal("Missing required parameter : filename")
+	}
+	file, diags := hclsyntax.ParseConfig(src, filename, hcl.Pos{Line: 1, Column: 1})
+	if diags.HasErrors() {
+		defaultLoggerSugar.Fatal("config parse", zap.Error(diags))
+	}
+
+	config := &internal.WeatherConfig{}
+
+	diags = gohcl.DecodeBody(file.Body, nil, config)
+	if diags.HasErrors() {
+		defaultLoggerSugar.Fatal("config parse", zap.Error(diags))
+	}
+
+	if config.OpenweatherSecret == "" {
+		defaultLoggerSugar.Fatal("Missing required parameter : openweather-secret")
+	}
+
+	addr := web.NewListenAddr("api.openweathermap.org", 443)
+
+	poller := poller.NewWeatherPoller(defaultLogger, addr, 45.75, 4.85, config.OpenweatherSecret)
+	// s3storage := storage.NewS3Storage(defaultLogger, bucket)
+	fileStorage := storage.NewFileStorage(defaultLogger, "db.json")
+
+	go func() {
+		for {
+			data := poller.Poll()
+			defer data.Close()
+
+			var buf bytes.Buffer
+			b, err := ioutil.ReadAll(io.TeeReader(data, &buf))
+			if err != nil {
+				defaultLogger.Sugar().Fatal("Wait next poll")
+			}
+
+			fileStorage.Store(bytes.NewReader(b))
+			// s3storage.Store(&buf)
+
+			defaultLogger.Sugar().Debug("Wait next poll")
+			time.Sleep(interval)
+		}
+	}()
+
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+
+	<-c
+
+	defaultLogger.Sugar().Info("shutting down")
+	os.Exit(0)
+}

cmd/weather/main.go

@@ -14,11 +14,6 @@ import (
 	"go.uber.org/zap"
 )
 
-const (
-	apiPrefix     = "/api"
-	apiRouterName = "api"
-)
-
 func main() {
 	var wait time.Duration
 	flag.DurationVar(&wait, "graceful-timeout", time.Second*15, "the duration for which the server gracefully wait for existing connections to finish - e.g. 15s or 1m")
@@ -26,7 +21,7 @@ func main() {
 
 	//logger
 	loggerLevel := zap.NewAtomicLevel()
-	defaultLogger := logger.NewLogger("default", "weather.log", loggerLevel)
+	defaultLogger := logger.NewLogger("weather", "weather.log", loggerLevel)
 	defer defaultLogger.Sync()
 
 	//http

config.sample.hcl

@@ -0,0 +1 @@
+openweather-secret = "<TO_COMPLETE>"

go.mod

@@ -3,19 +3,22 @@ module go/weather
 go 1.16
 
 require (
+	github.com/aws/aws-sdk-go v1.37.23
 	github.com/elastic/go-sysinfo v1.6.0 // indirect
 	github.com/elastic/go-windows v1.0.1 // indirect
 	github.com/gorilla/mux v1.8.0
-	github.com/hashicorp/go-version v1.2.1 // indirect
+	github.com/hashicorp/go-version v1.2.1
+	github.com/hashicorp/hcl v1.0.0
+	github.com/hashicorp/hcl/v2 v2.9.0
 	github.com/magefile/mage v1.11.0 // indirect
+	github.com/pelletier/go-toml v1.8.1
 	github.com/prometheus/procfs v0.6.0 // indirect
-	go.elastic.co/apm v1.11.0 // indirect
+	go.elastic.co/apm v1.11.0
-	go.elastic.co/apm/module/apmzap v1.11.0 // indirect
+	go.elastic.co/apm/module/apmzap v1.11.0
 	go.elastic.co/ecszap v1.0.0
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.16.0
 	golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46 // indirect
-	golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	howett.net/plist v0.0.0-20201203080718-1454fab16a06 // indirect
 )

go.sum

@@ -1,7 +1,17 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
+github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
+github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
+github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
+github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
+github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
+github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go v1.37.23 h1:bO80NcSmRv52w+GFpBegoLdlP/Z0OwUqQ9bbeCLCy/0=
+github.com/aws/aws-sdk-go v1.37.23/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/cucumber/godog v0.8.1 h1:lVb+X41I4YDreE+ibZ50bdXmySxgRviYFgKY6Aw4XE8=
 github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -12,24 +22,46 @@ github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6
 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
+github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
+github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/hcl/v2 v2.9.0 h1:7kJiMiKBqGHASbDJuFAMlpRMJLyhuLg/IsU/3EzwniA=
+github.com/hashicorp/hcl/v2 v2.9.0/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls=
 github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
+github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
+github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -43,12 +75,23 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis=
 github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.0 h1:DMOzIV76tmoDNE9pX6RSN0aDtCYeCg5VueieJaAo1uw=
 github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
+github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
+github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
+github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA=
+github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
 go.elastic.co/apm v1.11.0 h1:uJyt6nCW9880sZhfl1tB//Jy/5TadNoAd8edRUtgb3w=
 go.elastic.co/apm v1.11.0/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0=
 go.elastic.co/apm/module/apmzap v1.11.0 h1:KhAsnIorvxH9ar8tYV8jDM5wChaw5gnISdgInBwnomc=
@@ -73,29 +116,44 @@ go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM=
 go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46 h1:V066+OYJ66oTjnhm4Yrn7SXIwSCiDQJxpBxmvqb1N1c=
 golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -107,14 +165,17 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=

internal/config.go

@@ -0,0 +1,7 @@
+package internal
+
+//WeatherConfig weather application relative configuration file
+// this configuration is common to weather server and poller
+type WeatherConfig struct {
+	OpenweatherSecret string `hcl:"openweather-secret"`
+}

internal/poller/poller.go

@@ -0,0 +1,8 @@
+package poller
+
+import "io"
+
+//Poller represent interface to poll external api
+type Poller interface {
+	Poll() io.ReadCloser
+}

internal/poller/weather.go

@@ -0,0 +1,58 @@
+package poller
+
+import (
+	"go/weather/internal/web"
+	"go/weather/pkg/logger"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"go.uber.org/zap"
+)
+
+type weatherPoller struct {
+	client   *http.Client
+	endpoint *url.URL
+	logger   *logger.WeatherLogger
+}
+
+//NewWeatherPoller construct weather poller instance
+func NewWeatherPoller(log *logger.WeatherLogger, remote web.ListenAddr, lat, lon float64, secretID string) Poller {
+	u := &url.URL{
+		Scheme: "https",
+		Path:   "/data/2.5/onecall",
+		Host:   remote.String(),
+	}
+	q := u.Query()
+	q.Set("lat", strconv.FormatFloat(lat, 'f', 2, 64))
+	q.Set("lon", strconv.FormatFloat(lon, 'f', 2, 32))
+	q.Set("appid", secretID)
+	q.Set("units", "metric")
+	q.Set("lang", "fr")
+	u.RawQuery = q.Encode()
+
+	return &weatherPoller{
+		client:   http.DefaultClient,
+		endpoint: u,
+		logger:   log,
+	}
+}
+
+//Poll retrieve weather information
+func (w *weatherPoller) Poll() io.ReadCloser {
+	r, err := w.client.Get(w.endpoint.String())
+	w.logger.Debug("HTTP poll to openweathermap",
+		zap.Int("http.response.status_code", r.StatusCode),
+		zap.Int("http.response.body.bytes", int(r.ContentLength)),
+	)
+	w.handleError(err)
+	return r.Body
+}
+
+//handleError generic fatal error
+func (w *weatherPoller) handleError(err error) {
+	if err != nil {
+		w.logger.Fatal("Poller error", zap.Error(err))
+	}
+}

internal/storage/file.go

@@ -0,0 +1,54 @@
+package storage
+
+import (
+	"go/weather/pkg/logger"
+	"io"
+	"io/fs"
+	"os"
+
+	"go.uber.org/zap"
+)
+
+//s3Storage classe used to store oi.Reader to s3
+type fileStorage struct {
+	logger   *logger.WeatherLogger
+	filename string
+}
+
+//NewFileStorage instanciate storage object
+func NewFileStorage(log *logger.WeatherLogger, filename string) Storage {
+	s3 := fileStorage{
+		logger:   log,
+		filename: filename,
+	}
+	return &s3
+}
+
+//Store send data to s3 bucket
+func (ss *fileStorage) Store(content io.Reader) {
+	b, err := io.ReadAll(content)
+	ss.handleError(err)
+
+	err = AppendFile(ss.filename, append(b, "\n"...), fs.ModeAppend)
+	ss.handleError(err)
+}
+
+//AppendFile add line to an existing file
+func AppendFile(filename string, data []byte, perm fs.FileMode) error {
+	f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, perm)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(data)
+	if err1 := f.Close(); err1 != nil && err == nil {
+		err = err1
+	}
+	return err
+}
+
+//handleError generic fatal error
+func (ss *fileStorage) handleError(err error) {
+	if err != nil {
+		ss.logger.Fatal("Storage error", zap.Error(err))
+	}
+}

internal/storage/s3.go

@@ -0,0 +1,91 @@
+package storage
+
+import (
+	"fmt"
+	"go/weather/pkg/logger"
+	"io"
+	"os"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/defaults"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	toml "github.com/pelletier/go-toml"
+	"go.uber.org/zap"
+)
+
+//AwsDefaultSection toml default section
+type AwsDefaultSection struct {
+	// attribute should be public ! for go-toml
+	EndpointURL string `toml:"endpoint_url"`
+}
+
+//CustomAwsConfig custom toml config for aws
+type CustomAwsConfig struct {
+	Default AwsDefaultSection `toml:"default"`
+}
+
+//s3Storage classe used to store oi.Reader to s3
+type s3Storage struct {
+	logger  *logger.WeatherLogger
+	session *session.Session
+	bucket  string
+}
+
+//NewS3Storage instanciate storage object
+func NewS3Storage(log *logger.WeatherLogger, bucket string) Storage {
+	s3 := s3Storage{
+		logger: log,
+		bucket: bucket,
+	}
+
+	customConfig := CustomAwsConfig{}
+	err := decodeFile(defaults.SharedConfigFilename(), &customConfig)
+	s3.handleError(err)
+
+	s, err := session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+		Config: aws.Config{
+			Endpoint:   &customConfig.Default.EndpointURL,
+			DisableSSL: aws.Bool(true),
+			LogLevel:   aws.LogLevel(aws.LogDebug),
+		},
+	})
+	s3.handleError(err)
+	s3.session = s
+
+	return &s3
+}
+
+//Store send data to s3 bucket
+func (ss *s3Storage) Store(content io.Reader) {
+	uploader := s3manager.NewUploader(ss.session)
+
+	_, err := uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(ss.bucket),
+		Key:    aws.String(time.Now().UTC().Format(time.RFC3339)),
+		Body:   content,
+	})
+	fmt.Printf("failed to upload file, %v\n", err)
+	ss.handleError(err)
+}
+
+//DecodeFile call toml.Decode with file
+func decodeFile(fpath string, v interface{}) error {
+	bs, err := os.Open(fpath)
+	if err != nil {
+		return err
+	}
+	d := toml.NewDecoder(bs)
+	err = d.Decode(v)
+
+	return err
+}
+
+//handleError generic fatal error
+func (ss *s3Storage) handleError(err error) {
+	if err != nil {
+		ss.logger.Fatal("Storage error", zap.Error(err))
+	}
+}

internal/storage/storage.go

@@ -0,0 +1,8 @@
+package storage
+
+import "io"
+
+//Storage capacity
+type Storage interface {
+	Store(content io.Reader)
+}

internal/web/address.go

@@ -13,7 +13,7 @@ type listenAddr struct {
 }
 
 //NewListenAddr create new instance of ListenAddr
-func NewListenAddr(addr string, port int) listenAddr {
+func NewListenAddr(addr string, port int) ListenAddr {
 	return listenAddr{
 		Addr: addr,
 		Port: port,

internal/web/server.go

@@ -75,8 +75,6 @@ func (ws *WeatherServer) WithHandler() *WeatherServer {
 //WithHTTPLogging add http server request/response logging capability
 func (ws *WeatherServer) WithHTTPLogging() *WeatherServer {
 	ws.Server.Handler = ws.wlogger.HTTPLogHandler(ws.router)
-	// ws.router.Use(ws.wlogger.HTTPLogHandler)
-	// ws.router.Use(apmgorilla.Middleware())
 
 	api := ws.getOrCreateSubRouterByName(apiRouterName, apiPrefix)
 	api.HandleFunc("/log", ws.wlogger.Level.ServeHTTP)

pkg/logger/logger.go

@@ -89,14 +89,14 @@ func (wl *WeatherLogger) HTTPLogHandler(next http.Handler) http.Handler {
 		// span, ctx := apm.StartSpan(r.Context(), "updateRequestCount", "custom")
 		// defer span.End()
 
-		wl.logHTTPRequest(r)
+		wl.LogHTTPRequest(r)
 		next.ServeHTTP(&ww, r)
-		wl.logHTTPResponse(ww)
+		wl.LogHTTPResponse(ww)
 	})
 }
 
-//logHTTPRequest print request information
+//LogHTTPRequest print request information
-func (wl *WeatherLogger) logHTTPRequest(r *http.Request) {
+func (wl *WeatherLogger) LogHTTPRequest(r *http.Request) {
 	var buf bytes.Buffer
 	b, err := ioutil.ReadAll(io.TeeReader(r.Body, &buf))
 	wl.handleError(err)
@@ -126,8 +126,8 @@ func (wl *WeatherLogger) logHTTPRequest(r *http.Request) {
 	lwith.Debug("Request")
 }
 
-//logHTTPResponse print response information
+//LogHTTPResponse print response information
-func (wl *WeatherLogger) logHTTPResponse(ww ResponseWriter) {
+func (wl *WeatherLogger) LogHTTPResponse(ww ResponseWriter) {
 	wl.Logger.Debug("Response",
 		zap.Int("http.response.status_code", ww.Status),
 		zap.Int("http.response.body.bytes", ww.Length),