diff --git a/AGENTS.md b/AGENTS.md index a650dd4..301579b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,6 +88,8 @@ Fetch only the files relevant to the task. A typical example contains ## Examples +- **`alb-observability-and-alerting`** `[iaas, alb, waf, load-balancer, layer7, observability, metrics, alerting, log-alerts, grafana, tls]` + Ships the metrics and logs of a STACKIT Application Load Balancer into a STACKIT Observability instance and adds alert rules and a Grafana dashboard on top of them - **`alb-tls-examples`** `[alb, tls, certificate, load-balancer, lets-encrypt, iaas, ske]` A collection of STACKIT Application Load Balancer (ALB) showcases with different TLS strategies — from self-signed to Let's Encrypt, from a single VM to Kubernetes - **`cdn-s3-static-website`** `[cdn, s3, object-storage, static-website, waf]` diff --git a/examples/alb-observability-and-alerting/.terraform.lock.hcl b/examples/alb-observability-and-alerting/.terraform.lock.hcl new file mode 100644 index 0000000..adb8d5a --- /dev/null +++ b/examples/alb-observability-and-alerting/.terraform.lock.hcl @@ -0,0 +1,45 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.3.0" + constraints = ">= 4.0.0" + hashes = [ + "h1:5bCU/c+2HUh7GhclzNSH6gAuoCS4inW3obEtRAwu6WQ=", + "zh:0ab58d6f8991d436c7d2dbd89ed814709b949b07ac5a54ee53b0aec1fa772a8b", + "zh:60b347abcb56f45d97c56f14d895069cd15a83993f199777f571b79fea3642ee", + "zh:6889be32640349230de3f23856e6f04e0e9ced4a84a27d3f552fa54684448218", + "zh:73f8e1ecf7135033165fb14b7e8bf4d656f3ce13065ec35762ea0481975328c7", + "zh:94ce25ee253eca0b42cae9c856b36bca8103b6453012d1b279c3623c805f2d42", + "zh:96bc6de9fd67bc446fd11257872e1ffb1029a996ed1d65a3f6b43f6d408ad9ab", + "zh:97c609a310a51bfd504d704e036d72064a84bf0bdb36cc08cd4cc66098212b41", + "zh:a12c16e94533c5bd123f75032576b9dc91dd5d5ccd5f7cf331d0f2e1adc55cf8", + "zh:c4f014f876adf7af57188795050bda5b0029d8c7d7773031102b6c36dcf1fc21", + "zh:d9b0a21583aaa3df3a95394fb949a3c515ff71c2ff5a1fc4a73d364aa90bfca5", + "zh:da510d22f0c6d71ad19a76406f106b782448f512375787ecfabb338ed1e311a7", + "zh:f0e9447a9ce3a24cdaa113089e65663c836d8b9bfdb915a1c0284e0112cab5c0", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/stackitcloud/stackit" { + version = "0.113.0" + constraints = ">= 0.113.0" + hashes = [ + "h1:FJ/p+w2MRxFlEWsN3vNoO9mLJ3XpucMx5B0sn2mPvto=", + "zh:03efc3d7b2ec3b9248784a415a7655e3ac0220a4a8219623733e649b59003baa", + "zh:0dde99e7b343fa01f8eefc378171fb8621bedb20f59157d6cc8e3d46c738105f", + "zh:175dc58711203edfe05fb7f8a006c57ebd9d690ac4bfb41331b2bfb8871a57eb", + "zh:303a7738dcc57930526893caee6686ee0d58d8cfef9a0c51628434b98bb01b79", + "zh:3adb1fcfebd17ba0b891dc728ac2f7e7f11f69d70dc1164361492ebf6aa6005b", + "zh:3c329196b67c469acb1ebbcc6616d0385a06d1e1991a308a4e7f59a952db5669", + "zh:429bbd6126629f515fb3be163757bb9f9aca14aea4a123a44d7abc054d35b97a", + "zh:52262dec7ad53bc2c9934471f8bf530e2a0dc5abcbc9fbee927abe7b1453b5a1", + "zh:872b69f652350da65ba512ed01844d08ddc556148638312782ea8f4686dc93f2", + "zh:9fef91d4d809d3e8785ecafcd14553ef1e008408745fa7b44434b88a7349b754", + "zh:bcea168e976f956ad4d4b023e55f73d7fc1d1b33d316fd19183c9ecf6d07df78", + "zh:c0ebbf965bf8392d7aeb99fb3bb7d9910e48261d69dc2ac1e6757c51a0a26a1c", + "zh:dc8f7972f9141bddb050f80759b971a73a89120f4a26449ee54510897cf1e4c6", + "zh:f48f1080e7fb113909c751a7faef668ce543d78b643b02d43880c88d660e9a50", + ] +} diff --git a/examples/alb-observability-and-alerting/010-provider.tf b/examples/alb-observability-and-alerting/010-provider.tf new file mode 100644 index 0000000..7f464d4 --- /dev/null +++ b/examples/alb-observability-and-alerting/010-provider.tf @@ -0,0 +1,34 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5.0" + required_providers { + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.113.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0.0" + } + } +} + +provider "stackit" { + default_region = var.stackit_region + service_account_key_path = var.stackit_service_account_key_path + # required for the stackit_image_v2 data source + enable_beta_resources = true +} diff --git a/examples/alb-observability-and-alerting/020-variables.tf b/examples/alb-observability-and-alerting/020-variables.tf new file mode 100644 index 0000000..b2066d6 --- /dev/null +++ b/examples/alb-observability-and-alerting/020-variables.tf @@ -0,0 +1,140 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "stackit_project_id" { + description = "The STACKIT project ID to deploy resources into." + type = string + + validation { + condition = can(regex("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$", var.stackit_project_id)) + error_message = "The stackit_project_id must be a valid UUID." + } +} + +variable "stackit_region" { + description = "The STACKIT region to deploy resources into." + type = string + default = "eu01" +} + +variable "stackit_service_account_key_path" { + description = "Path to the STACKIT service account key JSON file used for provider authentication." + type = string +} + +variable "name_prefix" { + description = "Prefix applied to the names of all resources. Lowercase letters, digits and hyphens only." + type = string + default = "alb-obs" + + validation { + condition = can(regex("^[a-z0-9]+(-[a-z0-9]+)*$", var.name_prefix)) && length(var.name_prefix) <= 20 + error_message = "The name_prefix must be 1-20 characters of lowercase letters, digits and single hyphens, starting and ending with a letter or digit." + } +} + +variable "network_cidr" { + description = "IPv4 prefix of the private network that hosts the backends and the load balancer." + type = string + default = "10.20.0.0/24" + + validation { + condition = can(cidrnetmask(var.network_cidr)) + error_message = "The network_cidr must be a valid IPv4 CIDR, e.g. 10.20.0.0/24." + } +} + +variable "availability_zones" { + description = "Availability zones for the backend VMs. One VM is created per zone." + type = list(string) + default = ["eu01-1", "eu01-2"] + + validation { + condition = length(var.availability_zones) >= 1 && length(var.availability_zones) <= 3 + error_message = "Provide between one and three availability zones." + } +} + +variable "machine_type" { + description = "Machine type of the backend VMs." + type = string + default = "c2i.1" +} + +variable "image_name" { + description = "Name of the boot image for the backend VMs, resolved via the stackit_image_v2 data source. The image must ship python3." + type = string + default = "Debian 12" +} + +variable "boot_volume_size_gb" { + description = "Boot volume size of each backend VM in GB." + type = number + default = 20 +} + +variable "alb_plan_id" { + description = "Service plan of the Application Load Balancer. p10 is the smallest plan." + type = string + default = "p10" +} + +variable "alb_allowed_source_ranges" { + description = "Source CIDRs that may reach the load balancer listeners." + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "observability_plan_name" { + description = "Service plan of the Observability instance. Logs and log alerts require an Observability-* plan, not an Observability-Monitoring-* plan. The plan also caps the metric samples per minute the load balancer may push; choose a larger plan if metrics arrive with gaps." + type = string + default = "Observability-Starter-EU01" +} + +variable "logs_retention_days" { + description = "Retention of the load balancer logs in the Observability instance." + type = number + default = 7 +} + +variable "metrics_retention_days" { + description = "Retention of the load balancer metrics in the Observability instance." + type = number + default = 90 +} + +variable "alert_email" { + description = "Email address that receives alert notifications. Leave unset to deploy the alert rules without a notification receiver." + type = string + default = null +} + +variable "alert_webhook_url" { + description = "Webhook URL that receives alert notifications. Leave unset to deploy the alert rules without a notification receiver." + type = string + default = null + sensitive = true +} + +variable "alert_traffic_min_bytes_per_second" { + description = "Throughput in bytes per second that the current throughput (spike alert) or the throughput of the previous hour (drop alert) must exceed before the traffic alerts fire, so that idle load balancers do not alert." + type = number + default = 100000 +} + +variable "alert_waf_blocks_per_5m" { + description = "Number of WAF-blocked requests within five minutes above which the WAF alert fires." + type = number + default = 10 +} diff --git a/examples/alb-observability-and-alerting/030-locals.tf b/examples/alb-observability-and-alerting/030-locals.tf new file mode 100644 index 0000000..78692c6 --- /dev/null +++ b/examples/alb-observability-and-alerting/030-locals.tf @@ -0,0 +1,57 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + labels = { + example = "alb-observability-and-alerting" + managed-by = "terraform" + } + + # One backend VM per availability zone, keyed by a two-digit index. + backends = { for idx, az in var.availability_zones : format("%02d", idx + 1) => az } + + backend_port = 8080 + target_pool_name = "${var.name_prefix}-backends" + alb_name = "${var.name_prefix}-alb" + + # Label selector that scopes the PromQL expressions to this load balancer. + # The load balancer reports one Envoy cluster per target pool and listener + # plus an internal xds_cluster, which is excluded. + lb_selector = "{stackit_lb_name=\"${local.alb_name}\", envoy_cluster_name!=\"xds_cluster\"}" + + # Alertmanager configuration of the Observability instance. Rules are always + # deployed; notifications are only routed when at least one receiver is set. + # The webhook URL is a sensitive variable; comparing it with null would mark + # the whole alert_config as sensitive and hide it from the plan output, so + # only the presence flag is unmasked. The URL itself stays redacted. + webhook_set = nonsensitive(var.alert_webhook_url != null) + alerting_enabled = var.alert_email != null || local.webhook_set + + alert_config = local.alerting_enabled ? { + receivers = [ + { + name = "default" + email_configs = var.alert_email != null ? [{ to = var.alert_email }] : null + webhooks_configs = local.webhook_set ? [{ url = var.alert_webhook_url }] : null + } + ] + route = { + receiver = "default" + group_by = ["alertname"] + group_wait = "30s" + group_interval = "5m" + repeat_interval = "4h" + } + } : null +} diff --git a/examples/alb-observability-and-alerting/040-network.tf b/examples/alb-observability-and-alerting/040-network.tf new file mode 100644 index 0000000..b4d436b --- /dev/null +++ b/examples/alb-observability-and-alerting/040-network.tf @@ -0,0 +1,43 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "stackit_network" "this" { + project_id = var.stackit_project_id + name = "${var.name_prefix}-network" + ipv4_prefix = var.network_cidr + ipv4_nameservers = ["1.1.1.1", "9.9.9.9"] + labels = local.labels +} + +# The load balancer attaches its own target security group to the backend +# interfaces. That group only allows traffic from the load balancer, so the +# backends get a group of their own. A new security group permits all outbound +# traffic by default, which cloud-init needs to reach the metadata service. +resource "stackit_security_group" "backend" { + project_id = var.stackit_project_id + name = "${var.name_prefix}-backend" + description = "Backend VMs of the ${var.name_prefix} load balancer" + stateful = true + labels = local.labels +} + +resource "stackit_security_group_rule" "backend_http" { + project_id = var.stackit_project_id + security_group_id = stackit_security_group.backend.security_group_id + direction = "ingress" + description = "Backend port, reachable from inside the network" + protocol = { name = "tcp" } + port_range = { min = local.backend_port, max = local.backend_port } + ip_range = var.network_cidr +} diff --git a/examples/alb-observability-and-alerting/050-machines.tf b/examples/alb-observability-and-alerting/050-machines.tf new file mode 100644 index 0000000..f38afc9 --- /dev/null +++ b/examples/alb-observability-and-alerting/050-machines.tf @@ -0,0 +1,58 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "stackit_image_v2" "backend" { + project_id = var.stackit_project_id + name = var.image_name +} + +resource "stackit_network_interface" "backend" { + for_each = local.backends + + project_id = var.stackit_project_id + network_id = stackit_network.this.network_id + name = "${var.name_prefix}-backend-${each.key}" + security = true + + security_group_ids = [stackit_security_group.backend.security_group_id] + + # The load balancer adds its own target security group to the interface. + lifecycle { + ignore_changes = [security_group_ids] + } +} + +resource "stackit_server" "backend" { + for_each = local.backends + + project_id = var.stackit_project_id + name = "${var.name_prefix}-backend-${each.key}" + availability_zone = each.value + machine_type = var.machine_type + labels = local.labels + + boot_volume = { + source_type = "image" + source_id = data.stackit_image_v2.backend.image_id + size = var.boot_volume_size_gb + delete_on_termination = true + } + + network_interfaces = [stackit_network_interface.backend[each.key].network_interface_id] + + user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { + backend_port = local.backend_port + server_py = file("${path.module}/files/server.py") + }) +} diff --git a/examples/alb-observability-and-alerting/060-alb.tf b/examples/alb-observability-and-alerting/060-alb.tf new file mode 100644 index 0000000..fc52927 --- /dev/null +++ b/examples/alb-observability-and-alerting/060-alb.tf @@ -0,0 +1,133 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "tls_private_key" "this" { + algorithm = "RSA" + rsa_bits = 2048 +} + +resource "tls_self_signed_cert" "this" { + private_key_pem = tls_private_key.this.private_key_pem + + subject { + common_name = "${var.name_prefix}.example.internal" + organization = "STACKIT Example" + } + + validity_period_hours = 8760 + + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + ] +} + +resource "stackit_alb_certificate" "this" { + project_id = var.stackit_project_id + region = var.stackit_region + name = "${var.name_prefix}-certificate" + private_key = tls_private_key.this.private_key_pem + public_key = tls_self_signed_cert.this.cert_pem +} + +resource "stackit_public_ip" "alb" { + project_id = var.stackit_project_id + labels = local.labels + + lifecycle { + ignore_changes = [network_interface_id] + } +} + +resource "stackit_application_load_balancer" "this" { + project_id = var.stackit_project_id + region = var.stackit_region + name = local.alb_name + plan_id = var.alb_plan_id + external_address = stackit_public_ip.alb.ip + labels = local.labels + + networks = [ + { + network_id = stackit_network.this.network_id + role = "ROLE_LISTENERS_AND_TARGETS" + } + ] + + listeners = [ + { + name = "https" + port = 443 + protocol = "PROTOCOL_HTTPS" + http = { + hosts = [ + { + host = "*" + rules = [{ target_pool = local.target_pool_name }] + } + ] + } + https = { + certificate_config = { + certificate_ids = [stackit_alb_certificate.this.cert_id] + } + } + waf_config_name = stackit_alb_waf_configuration.this.name + } + ] + + target_pools = [ + { + name = local.target_pool_name + target_port = local.backend_port + targets = [ + for key, nic in stackit_network_interface.backend : { + display_name = "${var.name_prefix}-backend-${key}" + ip = nic.ipv4 + } + ] + active_health_check = { + interval = "5s" + interval_jitter = "1s" + timeout = "3s" + healthy_threshold = 2 + unhealthy_threshold = 2 + http_health_checks = { + path = "/healthz" + ok_status = ["200"] + } + } + } + ] + + options = { + private_network_only = false + access_control = { + allowed_source_ranges = var.alb_allowed_source_ranges + } + observability = { + logs = { + credentials_ref = stackit_loadbalancer_observability_credential.this.credentials_ref + push_url = stackit_observability_instance.this.logs_push_url + } + metrics = { + credentials_ref = stackit_loadbalancer_observability_credential.this.credentials_ref + push_url = stackit_observability_instance.this.metrics_push_url + } + } + } + + depends_on = [stackit_server.backend] +} diff --git a/examples/alb-observability-and-alerting/070-waf.tf b/examples/alb-observability-and-alerting/070-waf.tf new file mode 100644 index 0000000..033c757 --- /dev/null +++ b/examples/alb-observability-and-alerting/070-waf.tf @@ -0,0 +1,60 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Minimal WAF setup so that blocked requests show up in the logs. See +# examples/iaas-cross-az-layer7-loadbalancer-waf for the full WAF example. + +resource "stackit_alb_waf_managed_rule_set" "crs" { + project_id = var.stackit_project_id + name = "${var.name_prefix}-crs" + type = "TYPE_OWASP_CRS" +} + +resource "stackit_alb_waf_custom_rule_group" "this" { + project_id = var.stackit_project_id + name = "${var.name_prefix}-custom-rules" + + rules = [ + { + description = "Deny requests that carry the header X-Waf-Demo: block" + conditions = [ + { + variable = { + type = "VARIABLE_REQUEST_HEADERS" + value = "X-Waf-Demo" + } + operator = { + type = "OPERATOR_STREQ" + value = "block" + } + } + ] + behavior = { + action = "ACTION_DENY" + log = true + log_msg = "Custom rule: X-Waf-Demo header" + } + } + ] + + depends_on = [stackit_alb_waf_managed_rule_set.crs] +} + +resource "stackit_alb_waf_configuration" "this" { + project_id = var.stackit_project_id + name = "${var.name_prefix}-waf" + managed_rule_set_name = stackit_alb_waf_managed_rule_set.crs.name + custom_rule_group_name = stackit_alb_waf_custom_rule_group.this.name + labels = local.labels +} diff --git a/examples/alb-observability-and-alerting/080-observability.tf b/examples/alb-observability-and-alerting/080-observability.tf new file mode 100644 index 0000000..62050bf --- /dev/null +++ b/examples/alb-observability-and-alerting/080-observability.tf @@ -0,0 +1,41 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "stackit_observability_instance" "this" { + project_id = var.stackit_project_id + name = "${var.name_prefix}-observability" + plan_name = var.observability_plan_name + logs_retention_days = var.logs_retention_days + metrics_retention_days = var.metrics_retention_days + metrics_retention_days_5m_downsampling = var.metrics_retention_days + metrics_retention_days_1h_downsampling = var.metrics_retention_days + alert_config = local.alert_config +} + +# Technical credentials of the Observability instance. The load balancer uses +# them for basic authentication against the log and metric push endpoints. +resource "stackit_observability_credential" "alb" { + project_id = var.stackit_project_id + instance_id = stackit_observability_instance.this.instance_id + description = "Push credentials for the ${var.name_prefix} load balancer" +} + +# The load balancer service stores a copy of the credentials and exposes a +# reference that is set on the load balancer instead of the raw secret. +resource "stackit_loadbalancer_observability_credential" "this" { + project_id = var.stackit_project_id + display_name = "${var.name_prefix}-observability" + username = stackit_observability_credential.alb.username + password = stackit_observability_credential.alb.password +} diff --git a/examples/alb-observability-and-alerting/090-alerting.tf b/examples/alb-observability-and-alerting/090-alerting.tf new file mode 100644 index 0000000..109c893 --- /dev/null +++ b/examples/alb-observability-and-alerting/090-alerting.tf @@ -0,0 +1,153 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Metric based rules (PromQL) evaluated by the Observability instance. The +# load balancer exports health check results, connection counts and bytes per +# target pool, but no request counts, status codes or latencies. +resource "stackit_observability_alertgroup" "alb" { + project_id = var.stackit_project_id + instance_id = stackit_observability_instance.this.instance_id + name = "${var.name_prefix}-alb" + interval = "60s" + + rules = [ + { + record = "alb:healthy_targets:min" + expression = "min by (stackit_lb_name, envoy_cluster_name) (lb_healthcheck_targets_healthy${local.lb_selector})" + }, + { + record = "alb:throughput_bytes_per_second:rate5m" + expression = "sum by (stackit_lb_name, envoy_cluster_name) (rate(lb_target_pool_cx_rx_bytes_total${local.lb_selector}[5m]) + rate(lb_target_pool_cx_tx_bytes_total${local.lb_selector}[5m]))" + }, + { + alert = "AlbTargetPoolDegraded" + expression = "min by (stackit_lb_name, envoy_cluster_name) (lb_healthcheck_targets_healthy${local.lb_selector}) < ${length(local.backends)} and max by (stackit_lb_name, envoy_cluster_name) (lb_healthcheck_targets_healthy${local.lb_selector}) > 0" + for = "1m" + labels = { + severity = "warning" + } + annotations = { + summary = "At least one backend of the target pool fails its health check" + description = "Only {{ $value }} of ${length(local.backends)} backends of {{ $labels.envoy_cluster_name }} pass the active health check on the load balancer instance with the fewest healthy backends." + } + }, + { + alert = "AlbTargetPoolUnhealthy" + expression = "max by (stackit_lb_name, envoy_cluster_name) (lb_healthcheck_targets_healthy${local.lb_selector}) == 0" + for = "1m" + labels = { + severity = "critical" + } + annotations = { + summary = "No backend of the target pool passes its health check" + description = "No backend of {{ $labels.envoy_cluster_name }} passes the active health check any more." + } + }, + { + alert = "AlbHealthCheckFailures" + expression = "sum by (stackit_lb_name, envoy_cluster_name) (rate(lb_healthcheck_targets_failures_total${local.lb_selector}[5m])) > 0" + for = "5m" + labels = { + severity = "warning" + } + annotations = { + summary = "Health checks of the target pool keep failing" + description = "{{ $value | printf \"%.2f\" }} health checks per second fail for {{ $labels.envoy_cluster_name }}. A backend is down or answers /healthz with an error." + } + }, + { + alert = "AlbTrafficSpike" + expression = "sum by (stackit_lb_name) (rate(lb_target_pool_cx_rx_bytes_total${local.lb_selector}[5m]) + rate(lb_target_pool_cx_tx_bytes_total${local.lb_selector}[5m])) > 3 * sum by (stackit_lb_name) (rate(lb_target_pool_cx_rx_bytes_total${local.lb_selector}[1h] offset 5m) + rate(lb_target_pool_cx_tx_bytes_total${local.lb_selector}[1h] offset 5m)) and sum by (stackit_lb_name) (rate(lb_target_pool_cx_rx_bytes_total${local.lb_selector}[5m]) + rate(lb_target_pool_cx_tx_bytes_total${local.lb_selector}[5m])) > ${var.alert_traffic_min_bytes_per_second}" + for = "5m" + labels = { + severity = "warning" + } + annotations = { + summary = "Backend throughput is more than three times the throughput of the previous hour" + description = "The load balancer currently exchanges {{ $value | humanize }}B/s with its backends." + } + }, + { + alert = "AlbTrafficDrop" + expression = "sum by (stackit_lb_name) (rate(lb_target_pool_cx_rx_bytes_total${local.lb_selector}[5m]) + rate(lb_target_pool_cx_tx_bytes_total${local.lb_selector}[5m])) < 0.2 * sum by (stackit_lb_name) (rate(lb_target_pool_cx_rx_bytes_total${local.lb_selector}[1h] offset 5m) + rate(lb_target_pool_cx_tx_bytes_total${local.lb_selector}[1h] offset 5m)) and sum by (stackit_lb_name) (rate(lb_target_pool_cx_rx_bytes_total${local.lb_selector}[1h] offset 5m) + rate(lb_target_pool_cx_tx_bytes_total${local.lb_selector}[1h] offset 5m)) > ${var.alert_traffic_min_bytes_per_second}" + for = "5m" + labels = { + severity = "warning" + } + annotations = { + summary = "Backend throughput dropped below 20 % of the throughput of the previous hour" + description = "The load balancer currently exchanges {{ $value | humanize }}B/s with its backends." + } + }, + { + alert = "AlbMetricsAbsent" + expression = "absent(lb_healthcheck_targets_healthy{stackit_lb_name=\"${local.alb_name}\"})" + for = "10m" + labels = { + severity = "warning" + } + annotations = { + summary = "No metrics of the load balancer have been received for ten minutes" + description = "The Observability instance receives no metrics from the load balancer. Check the errors attribute of the load balancer and the sample limit of the Observability plan." + } + }, + { + alert = "ObservabilitySamplesRejected" + expression = "increase(instance_remote_write_samples_rejected_total[10m]) > 0" + for = "0s" + labels = { + severity = "warning" + } + annotations = { + summary = "The Observability instance rejected metric samples" + description = "{{ $value | humanize }} samples were rejected during the last ten minutes, usually because the sample limit of the plan was exceeded." + } + }, + ] +} + +# Log based rules (LogQL) evaluated against the WAF log stream of the load balancer. +resource "stackit_observability_logalertgroup" "waf" { + project_id = var.stackit_project_id + instance_id = stackit_observability_instance.this.instance_id + name = "${var.name_prefix}-waf" + interval = "60s" + + rules = [ + { + alert = "AlbWafBlockRateSpike" + expression = "sum(count_over_time({component=\"waf\", stackit_lb_name=\"${local.alb_name}\"} |= \"Access denied\" [5m])) > ${var.alert_waf_blocks_per_5m}" + for = "1m" + labels = { + severity = "warning" + } + annotations = { + summary = "The WAF blocked more than ${var.alert_waf_blocks_per_5m} requests within five minutes" + description = "{{ $value }} requests were denied by the WAF during the last five minutes. Check the WAF log stream for the rule IDs and client addresses." + } + }, + { + alert = "AlbWafCustomRuleTriggered" + expression = "sum(count_over_time({component=\"waf\", stackit_lb_name=\"${local.alb_name}\"} |= \"Custom rule: X-Waf-Demo header\" [5m])) > 0" + for = "0s" + labels = { + severity = "info" + } + annotations = { + summary = "The custom WAF rule denied a request" + description = "{{ $value }} requests matched the custom deny rule during the last five minutes." + } + }, + ] +} diff --git a/examples/alb-observability-and-alerting/100-outputs.tf b/examples/alb-observability-and-alerting/100-outputs.tf new file mode 100644 index 0000000..2cddf99 --- /dev/null +++ b/examples/alb-observability-and-alerting/100-outputs.tf @@ -0,0 +1,64 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "alb_external_address" { + description = "Public IPv4 address of the Application Load Balancer." + value = stackit_application_load_balancer.this.external_address +} + +output "alb_url" { + description = "HTTPS URL of the load balancer. The certificate is self-signed, use curl -k." + value = "https://${stackit_application_load_balancer.this.external_address}" +} + +output "backend_private_ips" { + description = "Private IPv4 addresses of the backend VMs, keyed by backend index." + value = { for key, nic in stackit_network_interface.backend : key => nic.ipv4 } +} + +output "observability_instance_id" { + description = "ID of the Observability instance that receives the load balancer metrics and logs." + value = stackit_observability_instance.this.instance_id +} + +output "grafana_url" { + description = "Grafana of the Observability instance. Import dashboards/alb-overview.json here." + value = stackit_observability_instance.this.grafana_url +} + +output "alerting_url" { + description = "Alertmanager of the Observability instance." + value = stackit_observability_instance.this.alerting_url +} + +output "metrics_url" { + description = "Prometheus compatible query endpoint of the Observability instance." + value = stackit_observability_instance.this.metrics_url +} + +output "logs_url" { + description = "Loki compatible query endpoint of the Observability instance." + value = stackit_observability_instance.this.logs_url +} + +output "observability_username" { + description = "Technical user of the Observability instance, also valid for the query endpoints." + value = stackit_observability_credential.alb.username +} + +output "observability_password" { + description = "Password of the technical user of the Observability instance." + value = stackit_observability_credential.alb.password + sensitive = true +} diff --git a/examples/alb-observability-and-alerting/MAINTAINERS.md b/examples/alb-observability-and-alerting/MAINTAINERS.md new file mode 100644 index 0000000..52c53ff --- /dev/null +++ b/examples/alb-observability-and-alerting/MAINTAINERS.md @@ -0,0 +1,10 @@ +# Maintainers + +General maintainers: + +- Tim Reibe (Tim.Reibe@digits.schwarz) +- Florian Schmidl (florian_michael.schmidl_ext@external.digits.schwarz) + +This example is actively maintained. The owner is responsible for reviewing and updating dependencies and functionalities on a monthly basis. +For questions, issues, or feature requests, please email general maintainers. +Please include the BP name and version in your request. We will track your request as an issue. diff --git a/examples/alb-observability-and-alerting/README.md b/examples/alb-observability-and-alerting/README.md new file mode 100644 index 0000000..6f5eee3 --- /dev/null +++ b/examples/alb-observability-and-alerting/README.md @@ -0,0 +1,306 @@ + + +# ALB Observability and Alerting + +Ships the metrics and logs of a STACKIT Application Load Balancer into a STACKIT Observability instance and adds alert rules and a Grafana dashboard on top of them. + +## Overview + +The other load balancer examples in this repository show how to terminate TLS and how to block traffic with the WAF, but not how to see what the load balancer is doing. This example closes that gap: + +- An Application Load Balancer (ALB) with an HTTPS listener, a self-signed certificate, an active HTTP health check and a minimal WAF configuration fronts two backend VMs in different availability zones. +- The ALB pushes its metrics (Prometheus remote write) and logs (Loki push API) to an Observability instance via `options.observability`. The push credentials are created by Terraform; the same technical user is exposed as outputs (`observability_username` and the sensitive `observability_password`) for the query commands in [Discovering metric and log names](#discovering-metric-and-log-names). +- The Observability instance evaluates PromQL alert rules (target pool health, health check failures, throughput anomalies, missing metrics) and LogQL alert rules (WAF block rate) and, optionally, routes notifications to an email address or a webhook. +- A Grafana dashboard in `dashboards/` visualises the same data. + +The backend VMs run a small HTTP application whose health, status code, delay and response size can be chosen per request, so every alert can be triggered on purpose. See [Testing](#testing). + +## Architecture + +```mermaid +flowchart LR + client([Client]) -- "HTTPS :443" --> alb + subgraph project["STACKIT project"] + subgraph net["Network 10.20.0.0/24"] + alb["Application Load Balancer
TLS termination · WAF · health checks"] + alb -- ":8080" --> b1["Backend 01
eu01-1"] + alb -- ":8080" --> b2["Backend 02
eu01-2"] + end + subgraph obs["Observability instance"] + loki[("Logs")] + prom[("Metrics")] + rules["Alert rules
PromQL · LogQL"] + grafana["Grafana
dashboard"] + am["Alertmanager"] + end + alb -- "Loki push" --> loki + alb -- "remote write" --> prom + loki --> rules + prom --> rules + loki --> grafana + prom --> grafana + rules --> am + end + am -. "email / webhook
(optional)" .-> receiver([Receiver]) +``` + +## What gets created + +| Component | Resource | Purpose | +| ---------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| Network | `stackit_network`, `stackit_security_group`, `stackit_security_group_rule` | Private network and a security group for the backends | +| Backends | `stackit_server`, `stackit_network_interface` (one per AZ) | Debian VMs, provisioned by cloud-init with the test application (`files/server.py`) on port 8080 | +| Certificate | `tls_private_key`, `tls_self_signed_cert`, `stackit_alb_certificate` | Self-signed certificate for the HTTPS listener | +| Load balancer | `stackit_public_ip`, `stackit_application_load_balancer` | HTTPS listener, target pool with active health check, log and metric shipping | +| WAF | `stackit_alb_waf_managed_rule_set`, `_custom_rule_group`, `_configuration` | OWASP Core Rule Set plus one custom deny rule (`X-Waf-Demo: block`) | +| Observability | `stackit_observability_instance`, `stackit_observability_credential` | Instance that stores logs and metrics, technical user for pushing and querying | +| Push credentials | `stackit_loadbalancer_observability_credential` | Copy of the technical user in the load balancer service, referenced by `credentials_ref` | +| Metric alerts | `stackit_observability_alertgroup` | Recording rules and PromQL alerts, see [Alert rules](#alert-rules) | +| Log alerts | `stackit_observability_logalertgroup` | LogQL alerts on the WAF log stream | + +The WAF configuration is intentionally minimal. See [`iaas-cross-az-layer7-loadbalancer-waf`](../iaas-cross-az-layer7-loadbalancer-waf/README.md) for the full WAF example. + +The backends need a security group of their own (`stackit_security_group.backend`): the load balancer attaches its target security group to the backend interfaces, and that group only permits traffic from the load balancer. Without an additional group with outbound rules the VMs cannot reach the metadata service and cloud-init never runs. + +## What the load balancer exports + +The load balancer pushes a fixed set of metrics and two log streams. This is what arrived in the Observability instance with provider 0.113.0 in August 2026: + +| Data | Names | Labels | +| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| Health checks | `lb_healthcheck_targets_healthy` (gauge), `lb_healthcheck_targets_failures_total` (counter) | `stackit_lb_name`, `stackit_lb_project`, `stackit_lb_vm` (one per load balancer instance), `envoy_cluster_name` | +| Backend connections | `lb_target_pool_cx_active` (gauge), `lb_target_pool_cx_total`, `lb_target_pool_cx_rx_bytes_total`, `lb_target_pool_cx_tx_bytes_total` (counters) | same | +| Envoy aliases | `envoy_cluster_health_check_healthy`, `envoy_cluster_health_check_failure`, `envoy_cluster_upstream_cx_active`, `envoy_cluster_upstream_cx_total`, `envoy_cluster_upstream_cx_rx_bytes_total`, `envoy_cluster_upstream_cx_tx_bytes_total` | same values as the `lb_*` metrics | +| WAF records | Log stream `{component="waf"}`, JSON lines with the Coraza record in the `message` field | `component`, `level`, `service_name`, `stackit_lb_name`, `stackit_lb_project` | +| Envoy system logs | Log stream `{component="envoy"}`, JSON lines with `level`, `logger`, `message` | same | + +`envoy_cluster_name` is `--`, for example `alb-obs-backends-HTTPS-443`; the internal `xds_cluster` also appears and is excluded by the rules. The load balancer does **not** export request counts, response status codes or latencies, and the Envoy stream contains no access logs. Alerts on error ratios or latency therefore cannot be built from the load balancer telemetry today; request-level information is only available for requests inspected by the WAF. The metrics of the Observability instance itself (`instance_*`, for example `instance_remote_write_samples_rejected_total` and `instance_alert_rules_plan`) are available in the same instance. + +## Prerequisites + +| Tool | Version | +| --------- | -------- | +| Terraform | >= 1.5.0 | +| curl, jq | any | + +A STACKIT service account key with the `editor` role on the target project is required. The project needs quota for one load balancer, one public IP, two VMs and one Observability instance. + +## Usage + +### 1. Configure variables + +```bash +cp terraform.tfvars.example terraform.tfvars +# fill in stackit_project_id and stackit_service_account_key_path +``` + +All other variables have defaults, see [`020-variables.tf`](020-variables.tf). Set `alert_email` or `alert_webhook_url` to receive notifications; without a receiver the alert rules are still evaluated and visible in Alertmanager and Grafana. + +### 2. Deploy + +```bash +terraform init +terraform apply +``` + +The apply takes ten to twelve minutes: the Observability instance needs seven to eight minutes, and the load balancer, which references the push URLs and credentials of the instance and is therefore created afterwards, another four. The backends need three to four more minutes after the apply to finish cloud-init and pass the health check; until then the load balancer answers `503 no healthy upstream`. + +### 3. Verify + +```bash +export ALB_URL=$(terraform output -raw alb_url) + +# the certificate is self-signed, hence -k +curl -k "$ALB_URL/" +curl -k "$ALB_URL/healthz" +``` + +Both requests return a JSON body with the name of the backend that answered. + +## Discovering metric and log names + +STACKIT documents that the load balancer is built on Envoy and that WAF records are shipped under the `component="waf"` log label, but it does not publish the names of the metrics the load balancer emits. The names in [What the load balancer exports](#what-the-load-balancer-exports) were read from a live instance with the commands below. Use them to confirm the names against your instance or to extend the rules: + +```bash +export OBS_USER=$(terraform output -raw observability_username) +export OBS_PASS=$(terraform output -raw observability_password) +export METRICS_URL=$(terraform output -raw metrics_url) +export LOGS_URL=$(terraform output -raw logs_url) + +# all metric names currently stored in the instance +curl -s -u "$OBS_USER:$OBS_PASS" "$METRICS_URL/api/v1/label/__name__/values" | jq -r '.data[]' + +# labels of the load balancer series +curl -s -u "$OBS_USER:$OBS_PASS" -G "$METRICS_URL/api/v1/series" \ + --data-urlencode 'match[]={__name__=~"lb_.*"}' | jq '.data' + +# labels used by the log streams and the values of the component label +curl -s -u "$OBS_USER:$OBS_PASS" "$LOGS_URL/loki/api/v1/labels" | jq +curl -s -u "$OBS_USER:$OBS_PASS" "$LOGS_URL/loki/api/v1/label/component/values" | jq + +# most recent WAF records, message field only +curl -s -u "$OBS_USER:$OBS_PASS" -G "$LOGS_URL/loki/api/v1/query_range" \ + --data-urlencode 'query={component="waf"} | json | line_format "{{.message}}"' --data-urlencode 'limit=20' \ + | jq -r '.data.result[].values[][1]' +``` + +Metrics arrive within a minute after the load balancer has been created. If the list of metric names stays empty, refresh the state and inspect the `errors` attribute of the load balancer; it is only re-read from the API during a plan, apply or refresh: + +```bash +terraform apply -refresh-only +terraform state show stackit_application_load_balancer.this | grep -A 3 errors +``` + +No output means the load balancer reports no errors. Entries of type `TYPE_METRICS_MISCONFIGURED` or `TYPE_LOGS_MISCONFIGURED` carry a description of what is wrong. + +## Alert rules + +All rules are evaluated every 60 seconds inside the Observability instance and are scoped to this load balancer via the `stackit_lb_name` label. Thresholds are variables, see [`020-variables.tf`](020-variables.tf). + +| Alert | Type | Condition | For | Severity | +| ------------------------------ | ------ | ------------------------------------------------------------------------------------------- | --- | -------- | +| `AlbTargetPoolDegraded` | PromQL | Fewer healthy backends than configured on at least one load balancer instance, but not none | 1m | warning | +| `AlbTargetPoolUnhealthy` | PromQL | No healthy backend left | 1m | critical | +| `AlbHealthCheckFailures` | PromQL | Health checks keep failing (`rate(lb_healthcheck_targets_failures_total[5m]) > 0`) | 5m | warning | +| `AlbTrafficSpike` | PromQL | Backend throughput above three times the throughput of the previous hour | 5m | warning | +| `AlbTrafficDrop` | PromQL | Backend throughput below 20 % of the throughput of the previous hour | 5m | warning | +| `AlbMetricsAbsent` | PromQL | No load balancer metrics received for ten minutes | 10m | warning | +| `ObservabilitySamplesRejected` | PromQL | The Observability instance rejected metric samples (plan limit) | 0s | warning | +| `AlbWafBlockRateSpike` | LogQL | More than `alert_waf_blocks_per_5m` requests denied by the WAF in 5 minutes | 1m | warning | +| `AlbWafCustomRuleTriggered` | LogQL | The custom deny rule matched at least once in 5 minutes | 0s | info | + +`AlbTrafficSpike` requires the current throughput and `AlbTrafficDrop` the throughput of the previous hour to exceed `alert_traffic_min_bytes_per_second` (default 100 kB/s), so an idle load balancer does not alert. Two recording rules, `alb:healthy_targets:min` and `alb:throughput_bytes_per_second:rate5m`, store the healthy backend count and the throughput as pre-aggregated series for ad-hoc queries. + +The Observability plan limits the number of rules in metric alert groups (`instance_alert_rules_plan`, 10 for `Observability-Starter-EU01`; current usage in `instance_alert_rules`). Recording rules count towards this limit, so this example uses nine of the ten; the two log alert rules are not counted. Changing a rule replaces the whole alert group. + +## Dashboard + +The dashboard in [`dashboards/alb-overview.json`](dashboards/alb-overview.json) shows healthy backends, health check failures, backend throughput and connections, denied WAF requests, WAF rule matches by rule ID, the sample usage of the Observability plan and the raw log streams. A `Load balancer` variable selects the load balancer by `stackit_lb_name`; set the `Configured backends` variable to the number of entries in `availability_zones` (default 2) so that the healthy backends stat turns orange when the pool is degraded. Import it once through the Grafana UI: + +1. Open the Grafana URL from `terraform output grafana_url` and sign in with your STACKIT account. +2. Go to **Dashboards → New → Import**. +3. Click **Upload dashboard JSON file** and select `dashboards/alb-overview.json`. +4. Select the data sources of the instance when prompted (`Thanos` for Prometheus, `Loki` for Loki) and click **Import**. + +The dashboard uses the built-in data sources of the instance, no additional configuration is needed. + +## Testing + +Every alert can be triggered from the command line. Allow one evaluation interval (60 s) plus the `for` duration of the rule before checking the alert state. + +### Baseline traffic + +```bash +export ALB_URL=$(terraform output -raw alb_url) + +# roughly 5 requests per second for about 10 minutes (each call includes a TLS handshake) +for i in $(seq 1 3000); do curl -sk -o /dev/null "$ALB_URL/"; sleep 0.1; done +``` + +The backend answers `/status/` with that status code and `/delay/` after that delay. Both are visible in the response, but not in the load balancer metrics, see [What the load balancer exports](#what-the-load-balancer-exports). + +### Target pool health + +```bash +# mark the backend that answers as unhealthy for five minutes; +# repeat until both backends have reported healthy: false +curl -k "$ALB_URL/healthz/fail" +curl -k "$ALB_URL/healthz/fail" +``` + +Each call marks the backend that happened to answer as unhealthy for five minutes (`/healthz/fail/` sets a different duration, at most 3600). The response body names the backend. After the unhealthy threshold of the active health check has been reached the load balancer stops routing to that backend, so the second call reaches the other one. + +Expected: `AlbTargetPoolDegraded` fires about two minutes after the first call, `AlbTargetPoolUnhealthy` about two minutes after the second, and `AlbHealthCheckFailures` after five minutes of failing checks. All three resolve on their own once the backends recover. While no backend is healthy the load balancer answers every request with `503 no healthy upstream`. + +To recover earlier, call `/healthz/ok` until every backend has appeared in a response: + +```bash +for i in $(seq 1 10); do curl -sk "$ALB_URL/healthz/ok"; done +``` + +A backend that is unhealthy while the other one is healthy receives no traffic and therefore cannot be reached this way; wait for it to recover. + +### WAF blocks + +```bash +# custom rule: denied by header match +curl -sk -o /dev/null -w '%{http_code}\n' -H 'X-Waf-Demo: block' "$ALB_URL/" + +# OWASP Core Rule Set: denied by anomaly score (SQL injection pattern) +curl -sk -o /dev/null -w '%{http_code}\n' "$ALB_URL/?id=1%27%20OR%20%271%27%3D%271" + +# more than alert_waf_blocks_per_5m blocked requests +for i in $(seq 1 20); do curl -sk -o /dev/null -H 'X-Waf-Demo: block' "$ALB_URL/"; done +``` + +Expected: both single requests return `403`. `AlbWafCustomRuleTriggered` fires on the next evaluation, `AlbWafBlockRateSpike` about two minutes after the loop. + +### Throughput anomalies + +Both rules compare the current backend throughput with the throughput of the previous hour and only fire when the current throughput (spike) or the throughput of the previous hour (drop) exceeds `alert_traffic_min_bytes_per_second`. The backend answers `/bytes/` with a response of `n` bytes (at most 1 MiB), which makes it easy to move a lot of data with few requests. + +```bash +# spike: about 2 MB/s for twelve minutes +end=$((SECONDS + 720)) +while [ "$SECONDS" -lt "$end" ]; do + seq 1 4 | xargs -P 4 -I{} curl -sk -o /dev/null "$ALB_URL/bytes/500000" + sleep 1 +done +``` + +Expected: `AlbTrafficSpike` fires about seven minutes into the loop (the five-minute rate window has to fill up, then the rule waits five minutes; with no traffic in the previous hour the threshold is reached earlier). `AlbTrafficDrop` fires about ten minutes after the loop has stopped, because the previous hour still contains the spike, and resolves once that hour has passed. + +### Alert state + +```bash +export ALERTING_URL=$(terraform output -raw alerting_url) + +# active alerts in Alertmanager, metric and log alerts alike +curl -s -u "$OBS_USER:$OBS_PASS" "$ALERTING_URL/api/v2/alerts" | jq -r '.[].labels.alertname' + +# evaluation state of the metric rules, including pending alerts +curl -s -u "$OBS_USER:$OBS_PASS" "$METRICS_URL/api/v1/rules" \ + | jq -r '.data.groups[].rules[] | select(.type == "alerting") | "\(.name)\t\(.state)"' +``` + +The same information is available in Grafana under **Alerting → Alert rules**; the dashboard shows the underlying metrics, not the alert state. + +### Correlating a blocked request with its log records + +Every WAF record carries a `unique_id` that is shared by all records of one request. List the denied requests, pick an ID and fetch everything the WAF logged for it: + +```bash +curl -s -u "$OBS_USER:$OBS_PASS" -G "$LOGS_URL/loki/api/v1/query_range" \ + --data-urlencode 'query={component="waf"} |= "Access denied" | json | line_format "{{.message}}"' --data-urlencode 'limit=5' \ + | jq -r '.data.result[].values[][1]' | grep -o 'unique_id "[^"]*"' + +curl -s -u "$OBS_USER:$OBS_PASS" -G "$LOGS_URL/loki/api/v1/query_range" \ + --data-urlencode 'query={component="waf"} |= "" | json | line_format "{{.message}}"' \ + | jq -r '.data.result[].values[][1]' +``` + +The records list the matched rule (`id`, `msg`, `file`), the request URI, the client address and the anomaly score that led to the block. The custom rule of this example has the rule ID `1000`, blocks by the Core Rule Set carry the ID `949111` next to the ID of the rule that raised the anomaly score. + +## Notes + +- The password of the technical user is stored in the Terraform state. Protect the state accordingly. +- `stackit_loadbalancer_observability_credential` cannot be updated in place. Rotating the Observability credential replaces it and updates the load balancer. +- The metric names and labels are not documented by STACKIT and may change. If your instance reports different names, use the commands in [Discovering metric and log names](#discovering-metric-and-log-names) to adapt the rules and the dashboard. +- The Observability plan caps the metric samples per minute (`instance_remote_write_samples_max_1m`, 5,000 for `Observability-Starter-EU01`). The load balancer pushes the metrics, so an exceeded limit is not visible as an HTTP 429 to you; it shows up as `ObservabilitySamplesRejected` firing and as gaps in the dashboard. A single load balancer of plan `p10` with two backends pushes about 80 samples per minute, roughly two percent of that limit. +- STACKIT documents the standard log stream of the load balancer as Envoy system logs (start, stop, health) and lists enhanced log delivery as a roadmap item. Once request logs or metrics become available, the `/status/` and `/delay/` endpoints of the backend can be used to test rules on them. + +## Cleanup + +```bash +terraform destroy +``` + +## References + +- [Application Load Balancer: basic concepts](https://docs.stackit.cloud/products/network/load-balancing-and-content-delivery/application-load-balancer/basics/basic-concepts-alb/) +- [Application Load Balancer WAF: basic concepts](https://docs.stackit.cloud/products/network/load-balancing-and-content-delivery/application-load-balancer/basics/basic-concepts-alb-waf/) +- [STACKIT Observability](https://docs.stackit.cloud/products/logging-and-monitoring/observability/) +- [Terraform provider: `stackit_application_load_balancer`](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/application_load_balancer) +- [Terraform provider: `stackit_loadbalancer_observability_credential`](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/loadbalancer_observability_credential) +- [Terraform provider: `stackit_observability_alertgroup`](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/observability_alertgroup) +- [Terraform provider: `stackit_observability_logalertgroup`](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/observability_logalertgroup) diff --git a/examples/alb-observability-and-alerting/cloud-init.yaml.tftpl b/examples/alb-observability-and-alerting/cloud-init.yaml.tftpl new file mode 100644 index 0000000..b9c2fd5 --- /dev/null +++ b/examples/alb-observability-and-alerting/cloud-init.yaml.tftpl @@ -0,0 +1,25 @@ +#cloud-config +write_files: + - path: /opt/backend/server.py + permissions: "0755" + content: | + ${indent(6, chomp(server_py))} + - path: /etc/systemd/system/backend.service + permissions: "0644" + content: | + [Unit] + Description=HTTP backend for load balancer tests + After=network-online.target + Wants=network-online.target + + [Service] + ExecStart=/usr/bin/python3 /opt/backend/server.py ${backend_port} + Restart=always + RestartSec=2 + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl daemon-reload + - systemctl enable --now backend.service diff --git a/examples/alb-observability-and-alerting/dashboards/alb-overview.json b/examples/alb-observability-and-alerting/dashboards/alb-overview.json new file mode 100644 index 0000000..99b1aec --- /dev/null +++ b/examples/alb-observability-and-alerting/dashboards/alb-overview.json @@ -0,0 +1,869 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Metrics data source of the Observability instance", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }, + { + "name": "DS_LOKI", + "label": "Loki", + "description": "Logs data source of the Observability instance", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "datasource", + "id": "loki", + "name": "Loki", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "logs", + "name": "Logs", + "version": "" + } + ], + "uid": "stackit-alb-overview", + "title": "STACKIT ALB Overview", + "description": "Backend health, throughput and WAF activity of a STACKIT Application Load Balancer", + "tags": ["stackit", "alb", "load-balancer", "waf"], + "editable": true, + "graphTooltip": 1, + "schemaVersion": 39, + "version": 1, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "refresh": "30s", + "templating": { + "list": [ + { + "name": "backends", + "label": "Configured backends", + "type": "textbox", + "query": "2", + "current": { + "selected": false, + "text": "2", + "value": "2" + }, + "options": [ + { + "selected": true, + "text": "2", + "value": "2" + } + ], + "hide": 0 + }, + { + "name": "lb", + "label": "Load balancer", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values(lb_healthcheck_targets_healthy, stackit_lb_name)", + "refId": "lb" + }, + "definition": "label_values(lb_healthcheck_targets_healthy, stackit_lb_name)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "options": [], + "sort": 1 + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "links": [], + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Healthy backends", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "min(lb_healthcheck_targets_healthy{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}) / $backends", + "legendFormat": "healthy" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value" + } + }, + { + "id": 2, + "type": "stat", + "title": "Health check failures / s", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(lb_healthcheck_targets_failures_total{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}[5m]))", + "legendFormat": "failures" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.01 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "value" + } + }, + { + "id": 3, + "type": "stat", + "title": "Active backend connections", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(lb_target_pool_cx_active{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"})", + "legendFormat": "active" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "value" + } + }, + { + "id": 4, + "type": "stat", + "title": "Backend throughput", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(lb_target_pool_cx_rx_bytes_total{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}[5m]) + rate(lb_target_pool_cx_tx_bytes_total{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}[5m]))", + "legendFormat": "bytes/s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "decimals": 1, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "value" + } + }, + { + "id": 5, + "type": "stat", + "title": "WAF denied (1h)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum(count_over_time({component=\"waf\", stackit_lb_name=~\"$lb\"} |= \"Access denied\" [1h]))", + "legendFormat": "denied" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value" + } + }, + { + "id": 6, + "type": "stat", + "title": "Sample limit used", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(instance_remote_write_samples_received_total[5m])) * 60 / max(instance_remote_write_samples_max_1m)", + "legendFormat": "used" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "none", + "textMode": "value" + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Healthy backends per load balancer instance", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "lb_healthcheck_targets_healthy{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}", + "legendFormat": "{{stackit_lb_vm}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 10, + "lineWidth": 2, + "lineInterpolation": "stepAfter" + }, + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Health check failures / s", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (stackit_lb_vm) (rate(lb_healthcheck_targets_failures_total{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}[5m]))", + "legendFormat": "{{stackit_lb_vm}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 20, + "lineWidth": 1, + "drawStyle": "bars" + }, + "decimals": 2, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Backend throughput", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(lb_target_pool_cx_rx_bytes_total{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}[5m]))", + "legendFormat": "from backends" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(lb_target_pool_cx_tx_bytes_total{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}[5m]))", + "legendFormat": "to backends" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 20, + "lineWidth": 1, + "stacking": { + "mode": "normal" + } + }, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Backend connections", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(lb_target_pool_cx_active{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"})", + "legendFormat": "active" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(lb_target_pool_cx_total{stackit_lb_name=~\"$lb\", envoy_cluster_name!=\"xds_cluster\"}[5m]))", + "legendFormat": "opened / s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 10, + "lineWidth": 1 + }, + "decimals": 2, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "WAF denied requests / min", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum(count_over_time({component=\"waf\", stackit_lb_name=~\"$lb\"} |= \"Access denied\" [1m]))", + "legendFormat": "denied" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "lineWidth": 1 + }, + "decimals": 0, + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "denied" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "WAF rule matches by rule ID (5 min)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (id) (count_over_time({component=\"waf\", stackit_lb_name=~\"$lb\"} | json | line_format \"{{.message}}\" | regexp `\\[id \"(?P[0-9]+)\"\\]` [5m]))", + "legendFormat": "rule {{id}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10, + "lineWidth": 1 + }, + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 13, + "type": "logs", + "title": "WAF records", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{component=\"waf\", stackit_lb_name=~\"$lb\"} | json | line_format \"{{.message}}\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + } + }, + { + "id": 14, + "type": "logs", + "title": "Load balancer system logs", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{component=\"envoy\", stackit_lb_name=~\"$lb\"} | json | line_format \"{{.level}} {{.logger}} {{.message}}\"" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + } + } + ] +} diff --git a/examples/alb-observability-and-alerting/files/server.py b/examples/alb-observability-and-alerting/files/server.py new file mode 100644 index 0000000..397a269 --- /dev/null +++ b/examples/alb-observability-and-alerting/files/server.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Minimal HTTP backend whose health, status code, delay and response size +# can be chosen per request, so that alert rules can be triggered on purpose. +# Usage: server.py [port] +import json +import socket +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8080 +HOSTNAME = socket.gethostname() +FAIL_SECONDS = 300 +STATE = {"unhealthy_until": 0.0} +LOCK = threading.Lock() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): + # Silence the per-request log of BaseHTTPRequestHandler; the health + # checks of the load balancer alone would add a line every few seconds. + pass + + def reply(self, status, body): + self.send_response(status) + self.send_header("X-Backend", HOSTNAME) + if status in (204, 304): + # these status codes must not carry a body + self.end_headers() + return + body["backend"] = HOSTNAME + payload = (json.dumps(body) + "\n").encode() + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(payload) + + def do_GET(self): + path = self.path.split("?", 1)[0].rstrip("/") or "/" + parts = path.strip("/").split("/") + if path == "/": + return self.reply(200, {"message": "hello"}) + if path == "/healthz": + with LOCK: + remaining = max(0, int(STATE["unhealthy_until"] - time.time())) + healthy = remaining == 0 + return self.reply( + 200 if healthy else 503, + {"healthy": healthy, "unhealthy_for_s": remaining}, + ) + if parts[0] == "healthz" and len(parts) >= 2 and parts[1] == "fail": + # unhealthy for FAIL_SECONDS, or for /healthz/fail/ + if len(parts) == 3 and parts[2].isdecimal(): + seconds = min(int(parts[2]), 3600) + elif len(parts) == 2: + seconds = FAIL_SECONDS + else: + return self.reply(404, {"error": "not found"}) + with LOCK: + STATE["unhealthy_until"] = time.time() + seconds + return self.reply(200, {"healthy": False, "unhealthy_for_s": seconds}) + if path == "/healthz/ok": + with LOCK: + STATE["unhealthy_until"] = 0.0 + return self.reply(200, {"healthy": True, "unhealthy_for_s": 0}) + if parts[0] == "status" and len(parts) == 2 and parts[1].isdecimal(): + status = int(parts[1]) + if 200 <= status <= 599: + return self.reply(status, {"status": status}) + if parts[0] == "delay" and len(parts) == 2 and parts[1].isdecimal(): + delay_ms = min(int(parts[1]), 30000) + time.sleep(delay_ms / 1000) + return self.reply(200, {"delay_ms": delay_ms}) + if parts[0] == "bytes" and len(parts) == 2 and parts[1].isdecimal(): + size = min(int(parts[1]), 1048576) + return self.reply(200, {"bytes": size, "payload": "x" * size}) + return self.reply(404, {"error": "not found"}) + + do_HEAD = do_GET + + +if __name__ == "__main__": + ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() diff --git a/examples/alb-observability-and-alerting/terraform.tfvars.example b/examples/alb-observability-and-alerting/terraform.tfvars.example new file mode 100644 index 0000000..0109c5e --- /dev/null +++ b/examples/alb-observability-and-alerting/terraform.tfvars.example @@ -0,0 +1,10 @@ +# Copy this file to terraform.tfvars and fill in your values. +# +# stackit_region defaults to "eu01" and can be omitted. + +stackit_project_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" +stackit_service_account_key_path = "/path/to/stackit-sa.json" + +# Optional: route alert notifications to an email address and/or a webhook. +# alert_email = "oncall@example.com" +# alert_webhook_url = "https://hooks.example.com/alerts"