example(alb): add observability and alerting example

AGENTS.md

@@ -88,6 +88,8 @@ Fetch only the files relevant to the task. A typical example contains
 
 ## Examples
 
+- **`alb-observability-and-alerting`** `[iaas, alb, waf, load-balancer, layer7, observability, metrics, alerting, log-alerts, grafana, tls]`  
+  Ships the metrics and logs of a STACKIT Application Load Balancer into a STACKIT Observability instance and adds alert rules and a Grafana dashboard on top of them
 - **`alb-tls-examples`** `[alb, tls, certificate, load-balancer, lets-encrypt, iaas, ske]`  
   A collection of STACKIT Application Load Balancer (ALB) showcases with different TLS strategies — from self-signed to Let's Encrypt, from a single VM to Kubernetes
 - **`cdn-s3-static-website`** `[cdn, s3, object-storage, static-website, waf]`  

examples/alb-observability-and-alerting/010-provider.tf

@@ -0,0 +1,34 @@
+# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+terraform {
+  required_version = ">= 1.5.0"
+  required_providers {
+    stackit = {
+      source  = "stackitcloud/stackit"
+      version = ">= 0.113.0"
+    }
+    tls = {
+      source  = "hashicorp/tls"
+      version = ">= 4.0.0"
+    }
+  }
+}
+
+provider "stackit" {
+  default_region           = var.stackit_region
+  service_account_key_path = var.stackit_service_account_key_path
+  # required for the stackit_image_v2 data source
+  enable_beta_resources = true
+}

examples/alb-observability-and-alerting/020-variables.tf

@@ -0,0 +1,140 @@
+# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+variable "stackit_project_id" {
+  description = "The STACKIT project ID to deploy resources into."
+  type        = string
+
+  validation {
+    condition     = can(regex("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$", var.stackit_project_id))
+    error_message = "The stackit_project_id must be a valid UUID."
+  }
+}
+
+variable "stackit_region" {
+  description = "The STACKIT region to deploy resources into."
+  type        = string
+  default     = "eu01"
+}
+
+variable "stackit_service_account_key_path" {
+  description = "Path to the STACKIT service account key JSON file used for provider authentication."
+  type        = string
+}
+
+variable "name_prefix" {
+  description = "Prefix applied to the names of all resources. Lowercase letters, digits and hyphens only."
+  type        = string
+  default     = "alb-obs"
+
+  validation {
+    condition     = can(regex("^[a-z0-9]+(-[a-z0-9]+)*$", var.name_prefix)) && length(var.name_prefix) <= 20
+    error_message = "The name_prefix must be 1-20 characters of lowercase letters, digits and single hyphens, starting and ending with a letter or digit."
+  }
+}
+
+variable "network_cidr" {
+  description = "IPv4 prefix of the private network that hosts the backends and the load balancer."
+  type        = string
+  default     = "10.20.0.0/24"
+
+  validation {
+    condition     = can(cidrnetmask(var.network_cidr))
+    error_message = "The network_cidr must be a valid IPv4 CIDR, e.g. 10.20.0.0/24."
+  }
+}
+
+variable "availability_zones" {
+  description = "Availability zones for the backend VMs. One VM is created per zone."
+  type        = list(string)
+  default     = ["eu01-1", "eu01-2"]
+
+  validation {
+    condition     = length(var.availability_zones) >= 1 && length(var.availability_zones) <= 3
+    error_message = "Provide between one and three availability zones."
+  }
+}
+
+variable "machine_type" {
+  description = "Machine type of the backend VMs."
+  type        = string
+  default     = "c2i.1"
+}
+
+variable "image_name" {
+  description = "Name of the boot image for the backend VMs, resolved via the stackit_image_v2 data source. The image must ship python3."
+  type        = string
+  default     = "Debian 12"
+}
+
+variable "boot_volume_size_gb" {
+  description = "Boot volume size of each backend VM in GB."
+  type        = number
+  default     = 20
+}
+
+variable "alb_plan_id" {
+  description = "Service plan of the Application Load Balancer. p10 is the smallest plan."
+  type        = string
+  default     = "p10"
+}
+
+variable "alb_allowed_source_ranges" {
+  description = "Source CIDRs that may reach the load balancer listeners."
+  type        = list(string)
+  default     = ["0.0.0.0/0"]
+}
+
+variable "observability_plan_name" {
+  description = "Service plan of the Observability instance. Logs and log alerts require an Observability-* plan, not an Observability-Monitoring-* plan. The plan also caps the metric samples per minute the load balancer may push; choose a larger plan if metrics arrive with gaps."
+  type        = string
+  default     = "Observability-Starter-EU01"
+}
+
+variable "logs_retention_days" {
+  description = "Retention of the load balancer logs in the Observability instance."
+  type        = number
+  default     = 7
+}
+
+variable "metrics_retention_days" {
+  description = "Retention of the load balancer metrics in the Observability instance."
+  type        = number
+  default     = 90
+}
+
+variable "alert_email" {
+  description = "Email address that receives alert notifications. Leave unset to deploy the alert rules without a notification receiver."
+  type        = string
+  default     = null
+}
+
+variable "alert_webhook_url" {
+  description = "Webhook URL that receives alert notifications. Leave unset to deploy the alert rules without a notification receiver."
+  type        = string
+  default     = null
+  sensitive   = true
+}
+
+variable "alert_traffic_min_bytes_per_second" {
+  description = "Throughput in bytes per second that the current throughput (spike alert) or the throughput of the previous hour (drop alert) must exceed before the traffic alerts fire, so that idle load balancers do not alert."
+  type        = number
+  default     = 100000
+}
+
+variable "alert_waf_blocks_per_5m" {
+  description = "Number of WAF-blocked requests within five minutes above which the WAF alert fires."
+  type        = number
+  default     = 10
+}

examples/alb-observability-and-alerting/030-locals.tf

@@ -0,0 +1,57 @@
+# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+locals {
+  labels = {
+    example    = "alb-observability-and-alerting"
+    managed-by = "terraform"
+  }
+
+  # One backend VM per availability zone, keyed by a two-digit index.
+  backends = { for idx, az in var.availability_zones : format("%02d", idx + 1) => az }
+
+  backend_port     = 8080
+  target_pool_name = "${var.name_prefix}-backends"
+  alb_name         = "${var.name_prefix}-alb"
+
+  # Label selector that scopes the PromQL expressions to this load balancer.
+  # The load balancer reports one Envoy cluster per target pool and listener
+  # plus an internal xds_cluster, which is excluded.
+  lb_selector = "{stackit_lb_name=\"${local.alb_name}\", envoy_cluster_name!=\"xds_cluster\"}"
+
+  # Alertmanager configuration of the Observability instance. Rules are always
+  # deployed; notifications are only routed when at least one receiver is set.
+  # The webhook URL is a sensitive variable; comparing it with null would mark
+  # the whole alert_config as sensitive and hide it from the plan output, so
+  # only the presence flag is unmasked. The URL itself stays redacted.
+  webhook_set      = nonsensitive(var.alert_webhook_url != null)
+  alerting_enabled = var.alert_email != null || local.webhook_set
+
+  alert_config = local.alerting_enabled ? {
+    receivers = [
+      {
+        name             = "default"
+        email_configs    = var.alert_email != null ? [{ to = var.alert_email }] : null
+        webhooks_configs = local.webhook_set ? [{ url = var.alert_webhook_url }] : null
+      }
+    ]
+    route = {
+      receiver        = "default"
+      group_by        = ["alertname"]
+      group_wait      = "30s"
+      group_interval  = "5m"
+      repeat_interval = "4h"
+    }
+  } : null
+}

examples/alb-observability-and-alerting/040-network.tf

@@ -0,0 +1,43 @@
+# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "stackit_network" "this" {
+  project_id       = var.stackit_project_id
+  name             = "${var.name_prefix}-network"
+  ipv4_prefix      = var.network_cidr
+  ipv4_nameservers = ["1.1.1.1", "9.9.9.9"]
+  labels           = local.labels
+}
+
+# The load balancer attaches its own target security group to the backend
+# interfaces. That group only allows traffic from the load balancer, so the
+# backends get a group of their own. A new security group permits all outbound
+# traffic by default, which cloud-init needs to reach the metadata service.
+resource "stackit_security_group" "backend" {
+  project_id  = var.stackit_project_id
+  name        = "${var.name_prefix}-backend"
+  description = "Backend VMs of the ${var.name_prefix} load balancer"
+  stateful    = true
+  labels      = local.labels
+}
+
+resource "stackit_security_group_rule" "backend_http" {
+  project_id        = var.stackit_project_id
+  security_group_id = stackit_security_group.backend.security_group_id
+  direction         = "ingress"
+  description       = "Backend port, reachable from inside the network"
+  protocol          = { name = "tcp" }
+  port_range        = { min = local.backend_port, max = local.backend_port }
+  ip_range          = var.network_cidr
+}

examples/alb-observability-and-alerting/050-machines.tf

@@ -0,0 +1,58 @@
+# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data "stackit_image_v2" "backend" {
+  project_id = var.stackit_project_id
+  name       = var.image_name
+}
+
+resource "stackit_network_interface" "backend" {
+  for_each = local.backends
+
+  project_id = var.stackit_project_id
+  network_id = stackit_network.this.network_id
+  name       = "${var.name_prefix}-backend-${each.key}"
+  security   = true
+
+  security_group_ids = [stackit_security_group.backend.security_group_id]
+
+  # The load balancer adds its own target security group to the interface.
+  lifecycle {
+    ignore_changes = [security_group_ids]
+  }
+}
+
+resource "stackit_server" "backend" {
+  for_each = local.backends
+
+  project_id        = var.stackit_project_id
+  name              = "${var.name_prefix}-backend-${each.key}"
+  availability_zone = each.value
+  machine_type      = var.machine_type
+  labels            = local.labels
+
+  boot_volume = {
+    source_type           = "image"
+    source_id             = data.stackit_image_v2.backend.image_id
+    size                  = var.boot_volume_size_gb
+    delete_on_termination = true
+  }
+
+  network_interfaces = [stackit_network_interface.backend[each.key].network_interface_id]
+
+  user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", {
+    backend_port = local.backend_port
+    server_py    = file("${path.module}/files/server.py")
+  })
+}