feature: public exposition ingress are shit
This commit is contained in:
parent
4919d4be98
commit
b64751a9a6
5
main.go
5
main.go
@ -21,17 +21,18 @@ func main() {
|
||||
Health: "/",
|
||||
},
|
||||
},
|
||||
Public: true,
|
||||
Env: map[string]string{
|
||||
"version": "1.0.0",
|
||||
},
|
||||
Dns: pulumi.StringRef("pulumi-test-nginx.localdomain"),
|
||||
//Dns: pulumi.StringRef("pulumi-test-nginx.localdomain"),
|
||||
AllowAllOrigin: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx.Export("name", nginxApplication.DeploymentName)
|
||||
ctx.Export("name", nginxApplication.ApplicationName)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
@ -2,29 +2,16 @@ package application
|
||||
|
||||
import (
|
||||
v1 "antoine-roux.tk/projects/go/pulumi-library/crds/kubernetes/certmanager/v1"
|
||||
dnsv1alpha1 "antoine-roux.tk/projects/go/pulumi-library/crds/kubernetes/externaldns/v1alpha1"
|
||||
traefikv1alpha1 "antoine-roux.tk/projects/go/pulumi-library/crds/kubernetes/traefik/v1alpha1"
|
||||
"antoine-roux.tk/projects/go/pulumi-library/pkg/exposition"
|
||||
"antoine-roux.tk/projects/go/pulumi-library/pkg/meta"
|
||||
"antoine-roux.tk/projects/go/pulumi-library/pkg/workload"
|
||||
"errors"
|
||||
"fmt"
|
||||
appsv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1"
|
||||
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||||
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
netv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/networking/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
"slices"
|
||||
)
|
||||
|
||||
const (
|
||||
loadBalancerAddressName = "internal-lb.localdomain"
|
||||
defaultRecordTTL = 180
|
||||
)
|
||||
|
||||
type ImagesConfiguration struct {
|
||||
Image string
|
||||
Path string
|
||||
Health string
|
||||
}
|
||||
|
||||
type Configuration struct {
|
||||
Name string
|
||||
Namespace string
|
||||
@ -33,162 +20,198 @@ type Configuration struct {
|
||||
Replicas *int
|
||||
Env map[string]string
|
||||
AllowAllOrigin bool
|
||||
Public bool
|
||||
}
|
||||
|
||||
type configuration struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Images []ImagesConfiguration
|
||||
Dns string
|
||||
Replicas int
|
||||
Env map[string]string
|
||||
|
||||
ShouldCreateDns bool
|
||||
ShouldCreateCertificate bool
|
||||
ShouldCreateIngress bool
|
||||
ResponseHeaders *traefikv1alpha1.MiddlewareSpecHeadersArgs
|
||||
type ImagesConfiguration struct {
|
||||
Image string
|
||||
Path string
|
||||
Health string
|
||||
}
|
||||
|
||||
type deploymentConfiguration struct {
|
||||
Env map[string]string
|
||||
Name string
|
||||
Replicas int
|
||||
ImageReference ImagesConfiguration
|
||||
type CreatedApplication struct {
|
||||
ApplicationName pulumi.StringOutput `pulumi:"application"`
|
||||
DeploymentName []pulumi.StringOutput `pulumi:"deployment"`
|
||||
}
|
||||
|
||||
type serviceConfiguration struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
type ingressConfiguration struct {
|
||||
Service *corev1.Service
|
||||
ImageReference ImagesConfiguration
|
||||
}
|
||||
|
||||
type Application struct {
|
||||
type application struct {
|
||||
pulumi.ResourceState
|
||||
|
||||
DeploymentName pulumi.StringOutput `pulumi:"deployment"`
|
||||
Name string
|
||||
Namespace string
|
||||
Services []service
|
||||
Dns string
|
||||
Replicas int
|
||||
Env map[string]string
|
||||
AllowAllOrigin bool
|
||||
Public bool
|
||||
|
||||
shouldCreateDns bool
|
||||
shouldCreateCertificate bool
|
||||
shouldCreateIngress bool
|
||||
}
|
||||
|
||||
func NewApplication(ctx *pulumi.Context, publicConfiguration *Configuration) (*Application, error) {
|
||||
type service struct {
|
||||
Image string
|
||||
Path string
|
||||
Health string
|
||||
}
|
||||
|
||||
if publicConfiguration.Name != "" && len(publicConfiguration.Images) > 0 {
|
||||
localConfiguration := &configuration{
|
||||
Name: publicConfiguration.Name,
|
||||
Namespace: publicConfiguration.Namespace,
|
||||
Env: publicConfiguration.Env,
|
||||
func NewApplication(ctx *pulumi.Context, configuration *Configuration) (*CreatedApplication, error) {
|
||||
|
||||
if configuration.Name != "" && len(configuration.Images) > 0 {
|
||||
|
||||
application := &application{
|
||||
Name: configuration.Name,
|
||||
Namespace: configuration.Namespace,
|
||||
Env: configuration.Env,
|
||||
AllowAllOrigin: configuration.AllowAllOrigin,
|
||||
Public: configuration.Public,
|
||||
}
|
||||
|
||||
var preventDuplicatePath []string
|
||||
for _, publicImageConfiguration := range publicConfiguration.Images {
|
||||
localImagesConfiguration := ImagesConfiguration{
|
||||
Image: publicImageConfiguration.Image,
|
||||
Path: publicImageConfiguration.Path,
|
||||
Health: publicImageConfiguration.Health,
|
||||
for _, image := range configuration.Images {
|
||||
serviceConfiguration := service{
|
||||
Image: image.Image,
|
||||
Path: image.Path,
|
||||
Health: image.Health,
|
||||
}
|
||||
if publicImageConfiguration.Path == "" {
|
||||
localImagesConfiguration.Path = "/"
|
||||
if image.Path == "" {
|
||||
serviceConfiguration.Path = "/"
|
||||
}
|
||||
if publicImageConfiguration.Health == "" {
|
||||
localImagesConfiguration.Health = "/"
|
||||
if image.Health == "" {
|
||||
serviceConfiguration.Health = "/"
|
||||
}
|
||||
if slices.Contains(preventDuplicatePath, localImagesConfiguration.Path) {
|
||||
return nil, errors.New("duplicate path in ingress configuration")
|
||||
if slices.Contains(preventDuplicatePath, serviceConfiguration.Path) {
|
||||
return nil, errors.New("duplicate path in ingress applicationConfiguration")
|
||||
}
|
||||
localConfiguration.Images = append(localConfiguration.Images, localImagesConfiguration)
|
||||
preventDuplicatePath = append(preventDuplicatePath, localImagesConfiguration.Path)
|
||||
application.Services = append(application.Services, serviceConfiguration)
|
||||
preventDuplicatePath = append(preventDuplicatePath, serviceConfiguration.Path)
|
||||
}
|
||||
|
||||
if publicConfiguration.Replicas != nil {
|
||||
localConfiguration.Replicas = *publicConfiguration.Replicas
|
||||
if configuration.Replicas != nil {
|
||||
application.Replicas = *configuration.Replicas
|
||||
} else {
|
||||
localConfiguration.Replicas = 1
|
||||
application.Replicas = 1
|
||||
}
|
||||
|
||||
if publicConfiguration.Dns != nil {
|
||||
localConfiguration.Dns = *publicConfiguration.Dns
|
||||
localConfiguration.ShouldCreateIngress = true
|
||||
localConfiguration.ShouldCreateDns = true
|
||||
localConfiguration.ShouldCreateCertificate = true
|
||||
if configuration.Dns != nil && configuration.Public {
|
||||
return nil, errors.New("public exposition and DNS are incompatible")
|
||||
}
|
||||
|
||||
if publicConfiguration.AllowAllOrigin {
|
||||
localConfiguration.ResponseHeaders = &traefikv1alpha1.MiddlewareSpecHeadersArgs{
|
||||
AccessControlAllowOriginList: toPulumiStringArray([]string{"*"}),
|
||||
}
|
||||
if configuration.Dns != nil {
|
||||
application.Dns = *configuration.Dns
|
||||
application.shouldCreateDns = true
|
||||
application.shouldCreateCertificate = true
|
||||
application.shouldCreateIngress = true
|
||||
}
|
||||
|
||||
return createResources(ctx, localConfiguration)
|
||||
if configuration.Public {
|
||||
application.shouldCreateDns = false
|
||||
application.shouldCreateCertificate = false
|
||||
application.shouldCreateIngress = true
|
||||
}
|
||||
|
||||
err := ctx.RegisterComponentResource("pkg:application:CreatedApplication", configuration.Name, application)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return application.createResources(ctx)
|
||||
} else {
|
||||
return nil, errors.New("missing required value Name or Image during generic application construction")
|
||||
return nil, errors.New("missing required value ApplicationName or Image during generic application construction")
|
||||
}
|
||||
}
|
||||
|
||||
func createResources(ctx *pulumi.Context, configuration *configuration, opts ...pulumi.ResourceOption) (*Application, error) {
|
||||
application := &Application{}
|
||||
err := ctx.RegisterComponentResource("pkg:application:Application", configuration.Name, application)
|
||||
func (application *application) createResources(ctx *pulumi.Context) (*CreatedApplication, error) {
|
||||
createdApplication := &CreatedApplication{
|
||||
ApplicationName: pulumi.String(application.Name).ToStringOutput(),
|
||||
}
|
||||
|
||||
namespaceConfiguration := &meta.NamespaceConfiguration{
|
||||
Name: application.Name,
|
||||
Namespace: application.Namespace,
|
||||
}
|
||||
namespace, err := namespaceConfiguration.CreateNamespace(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
namespace, err := createNamespace(ctx, configuration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ingressesParameter []ingressConfiguration
|
||||
var deployments []*appsv1.Deployment
|
||||
for index, image := range configuration.Images {
|
||||
indexedName := fmt.Sprintf("%s-%d", configuration.Name, index)
|
||||
var ingressServices []exposition.IngressServices
|
||||
|
||||
for index, service := range application.Services {
|
||||
indexedName := fmt.Sprintf("%s-%d", application.Name, index)
|
||||
appLabels := pulumi.StringMap{
|
||||
"app.kubernetes.io/name": pulumi.String(indexedName),
|
||||
}
|
||||
|
||||
deploymentParameter := &deploymentConfiguration{
|
||||
Name: indexedName,
|
||||
Env: configuration.Env,
|
||||
Replicas: configuration.Replicas,
|
||||
ImageReference: image,
|
||||
deploymentConfiguration := &workload.DeploymentConfiguration{
|
||||
Name: indexedName,
|
||||
Env: application.Env,
|
||||
Replicas: application.Replicas,
|
||||
ImageReference: &workload.ImageReference{
|
||||
Image: service.Image,
|
||||
Health: service.Health,
|
||||
},
|
||||
}
|
||||
deployment, err := createDeployment(ctx, deploymentParameter, namespace, appLabels, application)
|
||||
deployment, err := deploymentConfiguration.CreateDeployment(ctx, namespace, application, appLabels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
application.DeploymentName = deployment.Metadata.Name().Elem()
|
||||
createdApplication.DeploymentName = append(createdApplication.DeploymentName, deployment.Metadata.Name().Elem())
|
||||
|
||||
serviceParameter := serviceConfiguration{
|
||||
serviceConfiguration := exposition.ServiceConfiguration{
|
||||
Name: indexedName,
|
||||
}
|
||||
service, err := createService(ctx, serviceParameter, namespace, appLabels, application)
|
||||
createdService, err := serviceConfiguration.CreateService(ctx, namespace, application, appLabels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ingressesParameter = append(ingressesParameter, ingressConfiguration{
|
||||
Service: service,
|
||||
ImageReference: image,
|
||||
})
|
||||
|
||||
ingressServices = append(ingressServices,
|
||||
exposition.IngressServices{
|
||||
Service: createdService,
|
||||
Path: service.Path,
|
||||
})
|
||||
deployments = append(deployments, deployment)
|
||||
}
|
||||
|
||||
certificate, err := createCertificate(ctx, configuration, namespace, application)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if application.shouldCreateDns {
|
||||
dnsConfiguration := &exposition.DnsConfiguration{
|
||||
Name: application.Name,
|
||||
Dns: application.Dns,
|
||||
}
|
||||
_, err = dnsConfiguration.CreateDNSRecord(ctx, namespace, application)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
headerMiddleware, err := createMiddlewareAddResponseHeader(ctx, configuration, namespace, application)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, err = createIngress(ctx, configuration, namespace, certificate, ingressesParameter, application, headerMiddleware)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var certificate *v1.Certificate
|
||||
if application.shouldCreateCertificate {
|
||||
certificateConfiguration := &meta.CertificateConfiguration{
|
||||
Name: application.Name,
|
||||
Dns: application.Dns,
|
||||
}
|
||||
certificate, err = certificateConfiguration.CreateCertificate(ctx, namespace, application)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if application.shouldCreateIngress {
|
||||
ingressConfiguration := exposition.NewIngressConfiguration(
|
||||
application.Name,
|
||||
application.Dns,
|
||||
application.AllowAllOrigin,
|
||||
application.Public,
|
||||
ingressServices,
|
||||
)
|
||||
|
||||
_, err = createDNSRecord(ctx, configuration, namespace, application)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
err = ingressConfiguration.CreateIngress(ctx, namespace, application, certificate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
outs := pulumi.Map{}
|
||||
@ -201,247 +224,5 @@ func createResources(ctx *pulumi.Context, configuration *configuration, opts ...
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return application, nil
|
||||
}
|
||||
|
||||
func toPulumiStringArray(values []string) pulumi.StringArray {
|
||||
array := pulumi.StringArray{}
|
||||
for _, value := range values {
|
||||
array = append(array, pulumi.String(value))
|
||||
}
|
||||
return array
|
||||
}
|
||||
|
||||
func createMiddlewareAddResponseHeader(
|
||||
ctx *pulumi.Context,
|
||||
configuration *configuration,
|
||||
namespace *corev1.Namespace,
|
||||
application *Application,
|
||||
) (*traefikv1alpha1.Middleware, error) {
|
||||
if configuration.ResponseHeaders != nil {
|
||||
middlewareName := fmt.Sprintf("%s-response-header-middleware", configuration.Name)
|
||||
return traefikv1alpha1.NewMiddleware(ctx, middlewareName, &traefikv1alpha1.MiddlewareArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(configuration.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: &traefikv1alpha1.MiddlewareSpecArgs{
|
||||
Headers: configuration.ResponseHeaders,
|
||||
},
|
||||
}, pulumi.Parent(application))
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func createIngress(ctx *pulumi.Context, configuration *configuration, namespace *corev1.Namespace, certificate *v1.Certificate, ingressConfiguration []ingressConfiguration, application *Application, responseHeaderMiddleware *traefikv1alpha1.Middleware) (*netv1.Ingress, error) {
|
||||
if configuration.ShouldCreateIngress {
|
||||
host := pulumi.String(configuration.Dns)
|
||||
|
||||
var middlewares pulumi.StringInput
|
||||
if configuration.ResponseHeaders != nil {
|
||||
middlewares = pulumi.All(namespace.Metadata.Name().Elem(), responseHeaderMiddleware.Metadata.Name().Elem()).ApplyT(func(args []interface{}) string {
|
||||
return fmt.Sprintf("kube-ingress-gzip-compress@kubernetescrd,%s-%s@kubernetescrd", args[0], args[1])
|
||||
}).(pulumi.StringOutput)
|
||||
} else {
|
||||
middlewares = pulumi.String("kube-ingress-gzip-compress@kubernetescrd")
|
||||
}
|
||||
|
||||
ingressAnnotations := pulumi.StringMap{
|
||||
"traefik.ingress.kubernetes.io/router.middlewares": middlewares,
|
||||
"traefik.ingress.kubernetes.io/router.entrypoints": pulumi.String("websecure"),
|
||||
}
|
||||
|
||||
var ingressPaths netv1.HTTPIngressPathArray
|
||||
for _, service := range ingressConfiguration {
|
||||
ingressPaths = append(ingressPaths, netv1.HTTPIngressPathArgs{
|
||||
Path: pulumi.String(service.ImageReference.Path),
|
||||
PathType: pulumi.String("Prefix"),
|
||||
Backend: &netv1.IngressBackendArgs{
|
||||
Service: &netv1.IngressServiceBackendArgs{
|
||||
Name: service.Service.Metadata.Name().Elem(),
|
||||
Port: &netv1.ServiceBackendPortArgs{
|
||||
Name: pulumi.String("exposed-port"),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return netv1.NewIngress(ctx, configuration.Name, &netv1.IngressArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(configuration.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
Annotations: ingressAnnotations,
|
||||
},
|
||||
Spec: &netv1.IngressSpecArgs{
|
||||
IngressClassName: nil,
|
||||
Rules: &netv1.IngressRuleArray{
|
||||
netv1.IngressRuleArgs{
|
||||
Host: host,
|
||||
Http: &netv1.HTTPIngressRuleValueArgs{
|
||||
Paths: ingressPaths,
|
||||
},
|
||||
},
|
||||
},
|
||||
Tls: &netv1.IngressTLSArray{
|
||||
netv1.IngressTLSArgs{
|
||||
Hosts: pulumi.StringArray{host},
|
||||
SecretName: certificate.Spec.SecretName(),
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(application))
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func createCertificate(
|
||||
ctx *pulumi.Context, configuration *configuration, namespace *corev1.Namespace, application *Application,
|
||||
) (*v1.Certificate, error) {
|
||||
if configuration.ShouldCreateCertificate {
|
||||
return v1.NewCertificate(ctx, configuration.Name, &v1.CertificateArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(configuration.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: &v1.CertificateSpecArgs{
|
||||
SecretName: pulumi.String(fmt.Sprintf("%s-certificate", configuration.Name)),
|
||||
DnsNames: pulumi.StringArray{
|
||||
pulumi.String(configuration.Dns),
|
||||
},
|
||||
IssuerRef: &v1.CertificateSpecIssuerRefArgs{
|
||||
Name: pulumi.String("localdomain-issuer"),
|
||||
Kind: pulumi.String("ClusterIssuer"),
|
||||
Group: pulumi.String("cfssl-issuer.wikimedia.org"),
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(application))
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func createDNSRecord(ctx *pulumi.Context, configuration *configuration, namespace *corev1.Namespace, application *Application) (*dnsv1alpha1.DNSEndpoint, error) {
|
||||
if configuration.ShouldCreateDns {
|
||||
return dnsv1alpha1.NewDNSEndpoint(ctx, fmt.Sprintf("%s-record", configuration.Name), &dnsv1alpha1.DNSEndpointArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(configuration.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: &dnsv1alpha1.DNSEndpointSpecArgs{
|
||||
Endpoints: &dnsv1alpha1.DNSEndpointSpecEndpointsArray{
|
||||
&dnsv1alpha1.DNSEndpointSpecEndpointsArgs{
|
||||
DnsName: pulumi.String(configuration.Dns),
|
||||
RecordTTL: pulumi.Int(defaultRecordTTL),
|
||||
RecordType: pulumi.String("CNAME"),
|
||||
Targets: pulumi.StringArray{
|
||||
pulumi.String(loadBalancerAddressName),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(application))
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func createService(ctx *pulumi.Context, configuration serviceConfiguration, namespace *corev1.Namespace, appLabels pulumi.StringMap, application *Application) (*corev1.Service, error) {
|
||||
return corev1.NewService(ctx, fmt.Sprintf("%s-service", configuration.Name), &corev1.ServiceArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(configuration.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: &corev1.ServiceSpecArgs{
|
||||
Type: pulumi.String("ClusterIP"),
|
||||
Selector: appLabels,
|
||||
Ports: corev1.ServicePortArray{
|
||||
corev1.ServicePortArgs{
|
||||
Name: pulumi.String("exposed-port"),
|
||||
Port: pulumi.Int(8090),
|
||||
TargetPort: pulumi.String("http"),
|
||||
Protocol: pulumi.String("TCP"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(application))
|
||||
}
|
||||
|
||||
func createDeployment(
|
||||
ctx *pulumi.Context, configuration *deploymentConfiguration, namespace *corev1.Namespace, appLabels pulumi.StringMap, application *Application,
|
||||
) (*appsv1.Deployment, error) {
|
||||
env := corev1.EnvVarArray{}
|
||||
for key, value := range configuration.Env {
|
||||
env = append(env, &corev1.EnvVarArgs{
|
||||
Name: pulumi.String(key),
|
||||
Value: pulumi.String(value),
|
||||
})
|
||||
}
|
||||
|
||||
return appsv1.NewDeployment(ctx, fmt.Sprintf("%s-deployment", configuration.Name), &appsv1.DeploymentArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(configuration.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: appsv1.DeploymentSpecArgs{
|
||||
Selector: &metav1.LabelSelectorArgs{
|
||||
MatchLabels: appLabels,
|
||||
},
|
||||
Replicas: pulumi.Int(configuration.Replicas),
|
||||
Template: &corev1.PodTemplateSpecArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Labels: appLabels,
|
||||
},
|
||||
Spec: &corev1.PodSpecArgs{
|
||||
Containers: corev1.ContainerArray{
|
||||
corev1.ContainerArgs{
|
||||
Name: pulumi.String(configuration.Name),
|
||||
Image: pulumi.String(configuration.ImageReference.Image),
|
||||
Ports: corev1.ContainerPortArray{
|
||||
corev1.ContainerPortArgs{
|
||||
Name: pulumi.String("http"),
|
||||
ContainerPort: pulumi.Int(80),
|
||||
Protocol: pulumi.String("TCP"),
|
||||
},
|
||||
},
|
||||
Env: env,
|
||||
LivenessProbe: &corev1.ProbeArgs{
|
||||
HttpGet: &corev1.HTTPGetActionArgs{
|
||||
Path: pulumi.String(configuration.ImageReference.Health),
|
||||
Port: pulumi.Int(80),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(application))
|
||||
}
|
||||
|
||||
func createNamespace(ctx *pulumi.Context, configuration *configuration) (*corev1.Namespace, error) {
|
||||
return corev1.NewNamespace(ctx, fmt.Sprintf("%s-namespace", configuration.Name), &corev1.NamespaceArgs{
|
||||
Metadata: &metav1.ObjectMetaArgs{
|
||||
Name: pulumi.String(configuration.Namespace),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
})
|
||||
return createdApplication, nil
|
||||
}
|
||||
|
48
pkg/exposition/dns.go
Normal file
48
pkg/exposition/dns.go
Normal file
@ -0,0 +1,48 @@
|
||||
package exposition
|
||||
|
||||
import (
|
||||
"antoine-roux.tk/projects/go/pulumi-library/crds/kubernetes/externaldns/v1alpha1"
|
||||
"fmt"
|
||||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||||
v12 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
)
|
||||
|
||||
const (
|
||||
loadBalancerAddressName = "internal-lb.localdomain"
|
||||
defaultRecordTTL = 180
|
||||
)
|
||||
|
||||
type DnsConfiguration struct {
|
||||
Name string
|
||||
Dns string
|
||||
}
|
||||
|
||||
func (dns *DnsConfiguration) CreateDNSRecord(
|
||||
ctx *pulumi.Context,
|
||||
namespace *v1.Namespace,
|
||||
parentApplication pulumi.Resource,
|
||||
) (*v1alpha1.DNSEndpoint, error) {
|
||||
|
||||
return v1alpha1.NewDNSEndpoint(ctx, fmt.Sprintf("%s-record", dns.Name), &v1alpha1.DNSEndpointArgs{
|
||||
Metadata: &v12.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(dns.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: &v1alpha1.DNSEndpointSpecArgs{
|
||||
Endpoints: &v1alpha1.DNSEndpointSpecEndpointsArray{
|
||||
&v1alpha1.DNSEndpointSpecEndpointsArgs{
|
||||
DnsName: pulumi.String(dns.Dns),
|
||||
RecordTTL: pulumi.Int(defaultRecordTTL),
|
||||
RecordType: pulumi.String("CNAME"),
|
||||
Targets: pulumi.StringArray{
|
||||
pulumi.String(loadBalancerAddressName),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(parentApplication))
|
||||
}
|
213
pkg/exposition/ingress.go
Normal file
213
pkg/exposition/ingress.go
Normal file
@ -0,0 +1,213 @@
|
||||
package exposition
|
||||
|
||||
import (
|
||||
certManager "antoine-roux.tk/projects/go/pulumi-library/crds/kubernetes/certmanager/v1"
|
||||
traefik "antoine-roux.tk/projects/go/pulumi-library/crds/kubernetes/traefik/v1alpha1"
|
||||
"fmt"
|
||||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||||
meta "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
networking "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/networking/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
)
|
||||
|
||||
type IngressConfiguration struct {
|
||||
Name string
|
||||
Dns string
|
||||
Public bool
|
||||
ResponseHeaders *traefik.MiddlewareSpecHeadersArgs
|
||||
services []IngressServices
|
||||
}
|
||||
|
||||
type IngressServices struct {
|
||||
Service *v1.Service
|
||||
Path string
|
||||
}
|
||||
|
||||
func NewIngressConfiguration(name string, dns string, allowAllOrigin bool, public bool, services []IngressServices) *IngressConfiguration {
|
||||
ingressConfiguration := &IngressConfiguration{
|
||||
Name: name,
|
||||
Dns: dns,
|
||||
services: services,
|
||||
Public: public,
|
||||
}
|
||||
|
||||
if allowAllOrigin {
|
||||
ingressConfiguration.ResponseHeaders = &traefik.MiddlewareSpecHeadersArgs{
|
||||
AccessControlAllowOriginList: toPulumiStringArray([]string{"*"}),
|
||||
}
|
||||
}
|
||||
return ingressConfiguration
|
||||
}
|
||||
|
||||
func toPulumiStringArray(values []string) pulumi.StringArray {
|
||||
array := pulumi.StringArray{}
|
||||
for _, value := range values {
|
||||
array = append(array, pulumi.String(value))
|
||||
}
|
||||
return array
|
||||
}
|
||||
|
||||
func (ingress *IngressConfiguration) CreateIngress(
|
||||
ctx *pulumi.Context,
|
||||
namespace *v1.Namespace,
|
||||
parentApplication pulumi.Resource,
|
||||
certificate *certManager.Certificate,
|
||||
) error {
|
||||
|
||||
var middlewares pulumi.StringInput
|
||||
if ingress.ResponseHeaders != nil {
|
||||
headerMiddleware, err := ingress.createMiddlewareAddResponseHeader(ctx, namespace, parentApplication)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
middlewares = pulumi.All(namespace.Metadata.Name().Elem(), headerMiddleware.Metadata.Name().Elem()).ApplyT(func(args []interface{}) string {
|
||||
return fmt.Sprintf("kube-ingress-gzip-compress@kubernetescrd,%s-%s@kubernetescrd", args[0], args[1])
|
||||
}).(pulumi.StringOutput)
|
||||
} else {
|
||||
middlewares = pulumi.String("kube-ingress-gzip-compress@kubernetescrd")
|
||||
}
|
||||
|
||||
ingressAnnotations := pulumi.StringMap{
|
||||
"traefik.ingress.kubernetes.io/router.middlewares": middlewares,
|
||||
}
|
||||
|
||||
// https routing
|
||||
var ingressPaths networking.HTTPIngressPathArray
|
||||
for _, service := range ingress.services {
|
||||
ingressPaths = append(ingressPaths, networking.HTTPIngressPathArgs{
|
||||
Path: pulumi.String(service.Path),
|
||||
PathType: pulumi.String("Prefix"),
|
||||
Backend: &networking.IngressBackendArgs{
|
||||
Service: &networking.IngressServiceBackendArgs{
|
||||
Name: service.Service.Metadata.Name().Elem(),
|
||||
Port: &networking.ServiceBackendPortArgs{
|
||||
Name: pulumi.String("exposed-port"),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
var hosts pulumi.StringArray
|
||||
var certificateSecretName pulumi.StringOutput
|
||||
var namespaceName pulumi.StringPtrOutput
|
||||
if ingress.Public {
|
||||
ingressAnnotations["traefik.ingress.kubernetes.io/router.entrypoints"] = pulumi.String("exp-websecure")
|
||||
hosts = toPulumiStringArray([]string{"antoine-roux.tk", "antoineroux.tk", "www.antoine-roux.tk", "www.antoineroux.tk"})
|
||||
publicCertificate, err := certManager.GetCertificate(ctx, "nginxfront-certificate", pulumi.ID("default-public/nginxfront"), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
certificateSecretName = publicCertificate.Spec.SecretName()
|
||||
publicNamespace, err := v1.GetNamespace(ctx, "default-public", pulumi.ID("default-public"), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
namespaceName = publicNamespace.Metadata.Name()
|
||||
|
||||
} else {
|
||||
ingressAnnotations["traefik.ingress.kubernetes.io/router.entrypoints"] = pulumi.String("websecure")
|
||||
hosts = toPulumiStringArray([]string{ingress.Dns})
|
||||
certificateSecretName = certificate.Spec.SecretName()
|
||||
|
||||
// create http redirect to https
|
||||
err := ingress.createHttpRedirectIngress(ctx, namespace, parentApplication, ingressPaths)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
namespaceName = namespace.Metadata.Name()
|
||||
}
|
||||
|
||||
var ingressRules networking.IngressRuleArray
|
||||
for _, host := range hosts {
|
||||
ingressRules = append(ingressRules, networking.IngressRuleArgs{
|
||||
Host: host,
|
||||
Http: &networking.HTTPIngressRuleValueArgs{
|
||||
Paths: ingressPaths,
|
||||
},
|
||||
})
|
||||
}
|
||||
_, err := networking.NewIngress(ctx, fmt.Sprintf("%s-https", ingress.Name), &networking.IngressArgs{
|
||||
Metadata: &meta.ObjectMetaArgs{
|
||||
Namespace: namespaceName,
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(ingress.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
Annotations: ingressAnnotations,
|
||||
},
|
||||
Spec: &networking.IngressSpecArgs{
|
||||
IngressClassName: pulumi.String("traefik-internal"),
|
||||
Rules: &ingressRules,
|
||||
Tls: &networking.IngressTLSArray{
|
||||
networking.IngressTLSArgs{
|
||||
Hosts: hosts,
|
||||
SecretName: certificateSecretName,
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(parentApplication))
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (ingress *IngressConfiguration) createHttpRedirectIngress(
|
||||
ctx *pulumi.Context,
|
||||
namespace *v1.Namespace,
|
||||
parentApplication pulumi.Resource,
|
||||
paths networking.HTTPIngressPathArray,
|
||||
) error {
|
||||
|
||||
ingressAnnotations := pulumi.StringMap{
|
||||
"traefik.ingress.kubernetes.io/router.middlewares": pulumi.String("kube-ingress-gzip-compress@kubernetescrd,kube-ingress-redirect-scheme-https@kubernetescrd"),
|
||||
"traefik.ingress.kubernetes.io/router.entrypoints": pulumi.String("web"),
|
||||
}
|
||||
|
||||
_, err := networking.NewIngress(ctx, fmt.Sprintf("%s-http", ingress.Name), &networking.IngressArgs{
|
||||
Metadata: &meta.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(ingress.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
Annotations: ingressAnnotations,
|
||||
},
|
||||
Spec: &networking.IngressSpecArgs{
|
||||
IngressClassName: pulumi.String("traefik-internal"),
|
||||
Rules: &networking.IngressRuleArray{
|
||||
networking.IngressRuleArgs{
|
||||
Host: pulumi.String(ingress.Dns),
|
||||
Http: &networking.HTTPIngressRuleValueArgs{
|
||||
Paths: paths,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(parentApplication))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ingress *IngressConfiguration) createMiddlewareAddResponseHeader(
|
||||
ctx *pulumi.Context,
|
||||
namespace *v1.Namespace,
|
||||
parentApplication pulumi.Resource,
|
||||
) (*traefik.Middleware, error) {
|
||||
|
||||
middlewareName := fmt.Sprintf("%s-response-header-middleware", ingress.Name)
|
||||
return traefik.NewMiddleware(ctx, middlewareName, &traefik.MiddlewareArgs{
|
||||
Metadata: &meta.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(ingress.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: &traefik.MiddlewareSpecArgs{
|
||||
Headers: ingress.ResponseHeaders,
|
||||
},
|
||||
}, pulumi.Parent(parentApplication))
|
||||
}
|
41
pkg/exposition/service.go
Normal file
41
pkg/exposition/service.go
Normal file
@ -0,0 +1,41 @@
|
||||
package exposition
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||||
v12 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
)
|
||||
|
||||
type ServiceConfiguration struct {
|
||||
Name string
|
||||
}
|
||||
|
||||
func (service *ServiceConfiguration) CreateService(
|
||||
ctx *pulumi.Context,
|
||||
namespace *v1.Namespace,
|
||||
parentApplication pulumi.Resource,
|
||||
appLabels pulumi.StringMap,
|
||||
) (*v1.Service, error) {
|
||||
return v1.NewService(ctx, fmt.Sprintf("%s-service", service.Name), &v1.ServiceArgs{
|
||||
Metadata: &v12.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(service.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: &v1.ServiceSpecArgs{
|
||||
Type: pulumi.String("ClusterIP"),
|
||||
Selector: appLabels,
|
||||
Ports: v1.ServicePortArray{
|
||||
v1.ServicePortArgs{
|
||||
Name: pulumi.String("exposed-port"),
|
||||
Port: pulumi.Int(8090),
|
||||
TargetPort: pulumi.String("http"),
|
||||
Protocol: pulumi.String("TCP"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(parentApplication))
|
||||
}
|
41
pkg/meta/certificate.go
Normal file
41
pkg/meta/certificate.go
Normal file
@ -0,0 +1,41 @@
|
||||
package meta
|
||||
|
||||
import (
|
||||
certManager "antoine-roux.tk/projects/go/pulumi-library/crds/kubernetes/certmanager/v1"
|
||||
"fmt"
|
||||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||||
meta "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
)
|
||||
|
||||
type CertificateConfiguration struct {
|
||||
Name string
|
||||
Dns string
|
||||
}
|
||||
|
||||
func (certificate *CertificateConfiguration) CreateCertificate(
|
||||
ctx *pulumi.Context,
|
||||
namespace *v1.Namespace,
|
||||
parentApplication pulumi.Resource,
|
||||
) (*certManager.Certificate, error) {
|
||||
return certManager.NewCertificate(ctx, certificate.Name, &certManager.CertificateArgs{
|
||||
Metadata: &meta.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(certificate.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: &certManager.CertificateSpecArgs{
|
||||
SecretName: pulumi.String(fmt.Sprintf("%s-certificate", certificate.Name)),
|
||||
DnsNames: pulumi.StringArray{
|
||||
pulumi.String(certificate.Dns),
|
||||
},
|
||||
IssuerRef: &certManager.CertificateSpecIssuerRefArgs{
|
||||
Name: pulumi.String("localdomain-issuer"),
|
||||
Kind: pulumi.String("ClusterIssuer"),
|
||||
Group: pulumi.String("cfssl-issuer.wikimedia.org"),
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(parentApplication))
|
||||
}
|
24
pkg/meta/namespace.go
Normal file
24
pkg/meta/namespace.go
Normal file
@ -0,0 +1,24 @@
|
||||
package meta
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||||
v12 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
)
|
||||
|
||||
type NamespaceConfiguration struct {
|
||||
Name string
|
||||
Namespace string
|
||||
}
|
||||
|
||||
func (namespace *NamespaceConfiguration) CreateNamespace(ctx *pulumi.Context) (*v1.Namespace, error) {
|
||||
return v1.NewNamespace(ctx, fmt.Sprintf("%s-namespace", namespace.Name), &v1.NamespaceArgs{
|
||||
Metadata: &v12.ObjectMetaArgs{
|
||||
Name: pulumi.String(namespace.Namespace),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
79
pkg/workload/deployment.go
Normal file
79
pkg/workload/deployment.go
Normal file
@ -0,0 +1,79 @@
|
||||
package workload
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
v12 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1"
|
||||
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
|
||||
v13 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
|
||||
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
|
||||
)
|
||||
|
||||
type DeploymentConfiguration struct {
|
||||
Env map[string]string
|
||||
Name string
|
||||
Replicas int
|
||||
ImageReference *ImageReference
|
||||
}
|
||||
|
||||
type ImageReference struct {
|
||||
Image string
|
||||
Health string
|
||||
}
|
||||
|
||||
func (deployment *DeploymentConfiguration) CreateDeployment(
|
||||
ctx *pulumi.Context,
|
||||
namespace *v1.Namespace,
|
||||
parentApplication pulumi.Resource,
|
||||
appLabels pulumi.StringMap,
|
||||
) (*v12.Deployment, error) {
|
||||
env := v1.EnvVarArray{}
|
||||
for key, value := range deployment.Env {
|
||||
env = append(env, &v1.EnvVarArgs{
|
||||
Name: pulumi.String(key),
|
||||
Value: pulumi.String(value),
|
||||
})
|
||||
}
|
||||
|
||||
return v12.NewDeployment(ctx, fmt.Sprintf("%s-deployment", deployment.Name), &v12.DeploymentArgs{
|
||||
Metadata: &v13.ObjectMetaArgs{
|
||||
Namespace: namespace.Metadata.Name(),
|
||||
Labels: pulumi.StringMap{
|
||||
"app.kubernetes.io/part-of": pulumi.String(deployment.Name),
|
||||
"app.kubernetes.io/managed-by": pulumi.String("pulumi"),
|
||||
},
|
||||
},
|
||||
Spec: v12.DeploymentSpecArgs{
|
||||
Selector: &v13.LabelSelectorArgs{
|
||||
MatchLabels: appLabels,
|
||||
},
|
||||
Replicas: pulumi.Int(deployment.Replicas),
|
||||
Template: &v1.PodTemplateSpecArgs{
|
||||
Metadata: &v13.ObjectMetaArgs{
|
||||
Labels: appLabels,
|
||||
},
|
||||
Spec: &v1.PodSpecArgs{
|
||||
Containers: v1.ContainerArray{
|
||||
v1.ContainerArgs{
|
||||
Name: pulumi.String(deployment.Name),
|
||||
Image: pulumi.String(deployment.ImageReference.Image),
|
||||
Ports: v1.ContainerPortArray{
|
||||
v1.ContainerPortArgs{
|
||||
Name: pulumi.String("http"),
|
||||
ContainerPort: pulumi.Int(80),
|
||||
Protocol: pulumi.String("TCP"),
|
||||
},
|
||||
},
|
||||
Env: env,
|
||||
LivenessProbe: &v1.ProbeArgs{
|
||||
HttpGet: &v1.HTTPGetActionArgs{
|
||||
Path: pulumi.String(deployment.ImageReference.Health),
|
||||
Port: pulumi.Int(80),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, pulumi.Parent(parentApplication))
|
||||
}
|
Loading…
Reference in New Issue
Block a user