From 6349ce5108ed4f2efd01acbb1d4504fbc2403a28 Mon Sep 17 00:00:00 2001 From: RouxAntoine Date: Wed, 14 Jun 2023 07:51:58 +0200 Subject: [PATCH] feature: configure kafka topic and user with terraform --- Makefile | 19 ++++++++++++ manifest/.terraform.lock.hcl | 22 ++++++++++++++ manifest/kafka/input.tf | 15 +++++++++ manifest/kafka/topics.tf | 44 +++++++++++++++++++++++++++ manifest/kafka/users.tf | 59 ++++++++++++++++++++++++++++++++++++ manifest/main.tf | 50 ++++++++++++++++++++++++++++++ 6 files changed, 209 insertions(+) create mode 100644 Makefile create mode 100644 manifest/.terraform.lock.hcl create mode 100644 manifest/kafka/input.tf create mode 100644 manifest/kafka/topics.tf create mode 100644 manifest/kafka/users.tf create mode 100644 manifest/main.tf diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b154b41 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +ACTION=apply + +infrastructure-local: + cd manifest && \ + terraform workspace select kind-cluster-dev && \ + terraform $(ACTION) + +infrastructure-prod: + cd manifest && \ + terraform workspace select default && \ + terraform $(ACTION) + +topic-ssl: + kubectl view-secret -n streaming kafka-user user.p12 > user.p12 && \ + kubectl view-secret -n streaming cluster-development-cluster-ca-cert ca.crt | kcat -b kafka.127.0.0.1.nip.io:443 -L -J -X 'security.protocol=ssl' -X "ssl.ca.location=/dev/stdin" -X "ssl.keystore.location=user.p12" -X "ssl.keystore.password=$$(kubectl view-secret -n streaming kafka-user user.password)" |jq && \ + rm -rf user.p12 + +topic-scram: + kubectl view-secret -n streaming kafka-dev-listener-certificate ca.crt | kcat -b kafka.localdomain:9092 -L -J -X 'security.protocol=sasl_ssl' -X 'sasl.mechanism=SCRAM-SHA-512' -X 'sasl.username=kafka-user' -X "sasl.password=$(kubectl view-secret -n streaming kafka-user password)" -X "ssl.ca.location=/dev/stdin" | jq \ No newline at end of file diff --git a/manifest/.terraform.lock.hcl b/manifest/.terraform.lock.hcl new file mode 100644 index 0000000..179c181 --- /dev/null +++ b/manifest/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.20.0" + constraints = ">= 2.20.0" + hashes = [ + "h1:E7VAZorKe5oXn6h1nxP3ROwWNiQSrZlTawzix1sh8kM=", + "zh:30bc224c94d2c90a7d44554f2ad30e3b62c7ffc6ddb7d4fd31b9acafb8b5ad77", + "zh:3903cc9f0c3169a24265c4920d925ed7e37cbc4312237b29bd5b4ddcd6bdc535", + "zh:512240f6dad36c0116a8717487a4ea12a6b4191028782c5b6749037892e2c6ed", + "zh:57d5f77dcde7781803b465205aec3507780bfaa77031f5b893ae7cbebd4789b6", + "zh:6274ab8c3b59634c344c337218223640e9d954996b9299587ca924e4dfb77aa4", + "zh:6d838a25f3e3c696cf894f0adb44b41b461a2c76f914f1ae2c318ccbb1ec4e36", + "zh:92f09e3e03311c4e24601b704d85de57677f49e29f42cc3479fafa68f5de300a", + "zh:abb3cd606e485a46c076d6f60d37b5e5ecaa128c0150c8235627b484f2fac902", + "zh:afc07f5c0d7ce2cc907600e4f87a1290203a36221951e19e5d3f1409a0502377", + "zh:d9c01e4f12fabf5d6d9d11ceb409585b71c2abcad478496446de6ff18bbf2f5f", + "zh:f40faba2269184b305f229503945400ed6eeafec7ac395c23f243bccab7b11b2", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/manifest/kafka/input.tf b/manifest/kafka/input.tf new file mode 100644 index 0000000..b751e70 --- /dev/null +++ b/manifest/kafka/input.tf @@ -0,0 +1,15 @@ +variable "namespace" { + default = "default" + type = string + description = "namespace into which deploy kafka resource" +} + +variable "user-auth-type" { + type = string + description = "user authentication type : possible value scram-sha-512, tls, tls-external" +} + +variable "cluster-name" { + type = string + description = "kafka cluster name" +} diff --git a/manifest/kafka/topics.tf b/manifest/kafka/topics.tf new file mode 100644 index 0000000..d227052 --- /dev/null +++ b/manifest/kafka/topics.tf @@ -0,0 +1,44 @@ +resource "kubernetes_manifest" "receiver_topic" { + manifest = { + "apiVersion" = "kafka.strimzi.io/v1beta2" + "kind" = "KafkaTopic" + "metadata" = { + "labels" = { + "strimzi.io/cluster" = var.cluster-name + } + "name" = "dev.receiver.json" + "namespace" = var.namespace + } + "spec" = { + "config" = { + "retention.ms" = 5257000000 + "segment.bytes" = 1073741824 + } + "partitions" = 12 + "replicas" = 1 + } + } +} + +resource "kubernetes_manifest" "emitter_topic" { + manifest = { + "apiVersion" = "kafka.strimzi.io/v1beta2" + "kind" = "KafkaTopic" + "metadata" = { + "labels" = { + "strimzi.io/cluster" = var.cluster-name + } + "name" = "dev.emitter.json" + "namespace" = var.namespace + } + "spec" = { + "config" = { + # 2 months + "retention.ms" = 5257000000 + "segment.bytes" = 1073741824 + } + "partitions" = 12 + "replicas" = 1 + } + } +} \ No newline at end of file diff --git a/manifest/kafka/users.tf b/manifest/kafka/users.tf new file mode 100644 index 0000000..3f14604 --- /dev/null +++ b/manifest/kafka/users.tf @@ -0,0 +1,59 @@ +resource "kubernetes_manifest" "kafka_user" { + manifest = { + "apiVersion" = "kafka.strimzi.io/v1beta2" + "kind" = "KafkaUser" + "metadata" = { + "labels" = { + "strimzi.io/cluster" = var.cluster-name + } + "name" = "kafka-user" + "namespace" = var.namespace + } + "spec" = { + "authentication" = { + "type" = var.user-auth-type + } + "authorization" = { + "acls" = [ + { + "host" = "*" + "operations" = [ + "Describe", + "Write" + ] + "resource" = { + "name" = "dev.emitter." + "type" = "topic" + "patternType" = "prefix" + } + }, + { + "host" = "*" + "operations" = [ + "Describe", + "Read" + ] + "resource" = { + "name" = "dev.receiver." + "type" = "topic" + "patternType" = "prefix" + } + }, + { + "host" = "*" + "operations" = [ + "Describe", + "Read" + ] + "resource" = { + "type" = "group" + "name" = "some-consumer" + "patternType" = "literal" + } + } + ] + "type" = "simple" + } + } + } +} diff --git a/manifest/main.tf b/manifest/main.tf new file mode 100644 index 0000000..7ea3c38 --- /dev/null +++ b/manifest/main.tf @@ -0,0 +1,50 @@ +terraform { + required_version = ">= 1.0.4, < 2.0.0" + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20.0" + } + } + + backend "s3" { + endpoint = "http://s3.localdomain" + key = "projects/go-kafka.tfstate" + bucket = "terraform" + region = "FR" + skip_credentials_validation = true + skip_region_validation = true + skip_metadata_api_check = true + shared_credentials_file = "~/.aws/credentials" + force_path_style = true + } +} + +provider "kubernetes" { + config_paths = [ + "~/.kube/1-kind.kubeconfig", + "~/.kube/config.kubeconfig" + ] + + config_context = terraform.workspace == "default" ? "antoine@kubernetes" : terraform.workspace +} + +data "kubernetes_namespace_v1" "streaming_namespace" { + metadata { + name = "streaming" + } +} + +resource "kubernetes_namespace_v1" "application_namespace" { + metadata { + name = "some-application" + } +} + +module "kafka" { + source = "./kafka" + namespace = data.kubernetes_namespace_v1.streaming_namespace.metadata.0.name + user-auth-type = terraform.workspace == "default" ? "scram-sha-512" : "tls" + cluster-name = terraform.workspace == "default" ? "dev" : "cluster-development" +} \ No newline at end of file