diff --git a/AGENTS.md b/AGENTS.md index a650dd4..63e8a6b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,6 +88,8 @@ Fetch only the files relevant to the task. A typical example contains ## Examples +- **`alb-multi-tenant-routing`** `[iaas, alb, load-balancer, layer7, routing, tls, sni, ha, cross-az, canary, websocket]` + One STACKIT Application Load Balancer in front of several applications: two hostnames on a single HTTPS listener, path, header and query parameter rules, cookie persistence, WebSocket, and a target pool with its own health check per application - **`alb-tls-examples`** `[alb, tls, certificate, load-balancer, lets-encrypt, iaas, ske]` A collection of STACKIT Application Load Balancer (ALB) showcases with different TLS strategies — from self-signed to Let's Encrypt, from a single VM to Kubernetes - **`cdn-s3-static-website`** `[cdn, s3, object-storage, static-website, waf]` diff --git a/examples/alb-multi-tenant-routing/010-provider.tf b/examples/alb-multi-tenant-routing/010-provider.tf new file mode 100644 index 0000000..7f464d4 --- /dev/null +++ b/examples/alb-multi-tenant-routing/010-provider.tf @@ -0,0 +1,34 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5.0" + required_providers { + stackit = { + source = "stackitcloud/stackit" + version = ">= 0.113.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 4.0.0" + } + } +} + +provider "stackit" { + default_region = var.stackit_region + service_account_key_path = var.stackit_service_account_key_path + # required for the stackit_image_v2 data source + enable_beta_resources = true +} diff --git a/examples/alb-multi-tenant-routing/020-variables.tf b/examples/alb-multi-tenant-routing/020-variables.tf new file mode 100644 index 0000000..5ac04a1 --- /dev/null +++ b/examples/alb-multi-tenant-routing/020-variables.tf @@ -0,0 +1,119 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "stackit_project_id" { + description = "The STACKIT project ID to deploy resources into." + type = string + + validation { + condition = can(regex("^[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}$", var.stackit_project_id)) + error_message = "The stackit_project_id must be a valid UUID." + } +} + +variable "stackit_region" { + description = "The STACKIT region to deploy resources into." + type = string + default = "eu01" +} + +variable "stackit_service_account_key_path" { + description = "Path to the STACKIT service account key JSON file used for provider authentication." + type = string +} + +variable "name_prefix" { + description = "Prefix applied to the names of all resources. Lowercase letters, digits and hyphens only." + type = string + default = "alb-mt" + + validation { + condition = can(regex("^[a-z0-9]+(-[a-z0-9]+)*$", var.name_prefix)) && length(var.name_prefix) <= 20 + error_message = "The name_prefix must be 1-20 characters of lowercase letters, digits and single hyphens, starting and ending with a letter or digit." + } +} + +variable "domain" { + description = "Domain under which the applications are published. The listener routes app. and admin.. No DNS zone is created; the hostnames are resolved on the client with curl --resolve." + type = string + default = "example.internal" + + validation { + condition = can(regex("^[a-z0-9]+([-.][a-z0-9]+)*$", var.domain)) + error_message = "The domain must consist of lowercase letters, digits, hyphens and dots." + } +} + +variable "network_cidr" { + description = "IPv4 prefix of the private network that hosts the backends and the load balancer. In a project that belongs to a STACKIT Network Area the prefix must lie inside the network ranges of that area." + type = string + default = "10.20.1.0/24" + + validation { + condition = can(cidrnetmask(var.network_cidr)) + error_message = "The network_cidr must be a valid IPv4 CIDR, e.g. 10.20.1.0/24." + } +} + +variable "availability_zones" { + description = "Availability zones for the backend VMs. One VM is created per zone, and every VM is a target in every pool." + type = list(string) + default = ["eu01-1", "eu01-2"] + + validation { + condition = length(var.availability_zones) >= 1 && length(var.availability_zones) <= 3 + error_message = "Provide between one and three availability zones." + } +} + +variable "machine_type" { + description = "Machine type of the backend VMs." + type = string + default = "c2i.1" +} + +variable "image_name" { + description = "Name of the boot image for the backend VMs, resolved via the stackit_image_v2 data source. The image must ship python3." + type = string + default = "Debian 12" +} + +variable "boot_volume_size_gb" { + description = "Boot volume size of each backend VM in GB." + type = number + default = 20 +} + +variable "alb_plan_id" { + description = "Service plan of the Application Load Balancer. List the plans of your region with `stackit beta alb plans`; p10 is the only plan available at the time of writing." + type = string + default = "p10" +} + +variable "alb_allowed_source_ranges" { + description = "Source CIDRs that may reach the load balancer listener." + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "session_cookie_ttl" { + description = "Time-to-live of the persistence cookie the load balancer sets for the admin application, as whole seconds with an s suffix. 0s creates a session cookie that expires with the client session." + type = string + default = "300s" + + validation { + condition = can(regex("^[0-9]{1,8}s$", var.session_cookie_ttl)) + error_message = "The session_cookie_ttl must be a whole number of seconds followed by s, e.g. 300s or 0s." + } +} diff --git a/examples/alb-multi-tenant-routing/030-locals.tf b/examples/alb-multi-tenant-routing/030-locals.tf new file mode 100644 index 0000000..4ab66da --- /dev/null +++ b/examples/alb-multi-tenant-routing/030-locals.tf @@ -0,0 +1,61 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + labels = { + example = "alb-multi-tenant-routing" + managed-by = "terraform" + } + + # One backend VM per availability zone, keyed by a two-digit index. + backends = { for idx, az in var.availability_zones : format("%02d", idx + 1) => az } + + # The hostnames the listener routes. Both share the listener and the + # public IP; the load balancer selects the certificate by SNI. + app_host = "app.${var.domain}" + admin_host = "admin.${var.domain}" + + # One application per target pool. Every backend VM runs all of them, each + # on its own port, so every pool contains every VM and two machines are + # enough to make every route highly available. + applications = { + web = { port = 8081 } + api = { port = 8082 } + admin = { port = 8083 } + canary = { port = 8084 } + } + + pool_names = { for name, app in local.applications : name => "${var.name_prefix}-${name}" } + + # Command line of the backend application: one = pair per + # application, so that the port decides which pool a response reports. + backend_arguments = join(" ", [for name, app in local.applications : "${app.port}=${name}"]) + + backend_port_min = min([for app in values(local.applications) : app.port]...) + backend_port_max = max([for app in values(local.applications) : app.port]...) + + # Health check shared by all target pools. The backend answers /healthz + # with 200 while healthy and 503 after it has been told to fail. + active_health_check = { + interval = "5s" + interval_jitter = "1s" + timeout = "3s" + healthy_threshold = 2 + unhealthy_threshold = 2 + http_health_checks = { + path = "/healthz" + ok_status = ["200"] + } + } +} diff --git a/examples/alb-multi-tenant-routing/040-network.tf b/examples/alb-multi-tenant-routing/040-network.tf new file mode 100644 index 0000000..56f9436 --- /dev/null +++ b/examples/alb-multi-tenant-routing/040-network.tf @@ -0,0 +1,43 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "stackit_network" "this" { + project_id = var.stackit_project_id + name = "${var.name_prefix}-network" + ipv4_prefix = var.network_cidr + ipv4_nameservers = ["1.1.1.1", "9.9.9.9"] + labels = local.labels +} + +# The load balancer attaches its own target security group to the backend +# interfaces. That group only allows traffic from the load balancer, so the +# backends get a group of their own. A new security group permits all outbound +# traffic by default, which cloud-init needs to reach the metadata service. +resource "stackit_security_group" "backend" { + project_id = var.stackit_project_id + name = "${var.name_prefix}-backend" + description = "Backend VMs of the ${var.name_prefix} load balancer" + stateful = true + labels = local.labels +} + +resource "stackit_security_group_rule" "backend_http" { + project_id = var.stackit_project_id + security_group_id = stackit_security_group.backend.security_group_id + direction = "ingress" + description = "Application ports, reachable from inside the network" + protocol = { name = "tcp" } + port_range = { min = local.backend_port_min, max = local.backend_port_max } + ip_range = var.network_cidr +} diff --git a/examples/alb-multi-tenant-routing/050-machines.tf b/examples/alb-multi-tenant-routing/050-machines.tf new file mode 100644 index 0000000..d0e3a61 --- /dev/null +++ b/examples/alb-multi-tenant-routing/050-machines.tf @@ -0,0 +1,58 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "stackit_image_v2" "backend" { + project_id = var.stackit_project_id + name = var.image_name +} + +resource "stackit_network_interface" "backend" { + for_each = local.backends + + project_id = var.stackit_project_id + network_id = stackit_network.this.network_id + name = "${var.name_prefix}-backend-${each.key}" + security = true + + security_group_ids = [stackit_security_group.backend.security_group_id] + + # The load balancer adds its own target security group to the interface. + lifecycle { + ignore_changes = [security_group_ids] + } +} + +resource "stackit_server" "backend" { + for_each = local.backends + + project_id = var.stackit_project_id + name = "${var.name_prefix}-backend-${each.key}" + availability_zone = each.value + machine_type = var.machine_type + labels = local.labels + + boot_volume = { + source_type = "image" + source_id = data.stackit_image_v2.backend.image_id + size = var.boot_volume_size_gb + delete_on_termination = true + } + + network_interfaces = [stackit_network_interface.backend[each.key].network_interface_id] + + user_data = templatefile("${path.module}/cloud-init.yaml.tftpl", { + server_py = file("${path.module}/files/server.py") + backend_arguments = local.backend_arguments + }) +} diff --git a/examples/alb-multi-tenant-routing/060-certificates.tf b/examples/alb-multi-tenant-routing/060-certificates.tf new file mode 100644 index 0000000..a8874e8 --- /dev/null +++ b/examples/alb-multi-tenant-routing/060-certificates.tf @@ -0,0 +1,60 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# One self-signed certificate per hostname. Both are attached to the same +# HTTPS listener; the load balancer presents the one whose name matches the +# SNI of the TLS handshake. +locals { + certificate_hosts = { + app = local.app_host + admin = local.admin_host + } +} + +resource "tls_private_key" "this" { + for_each = local.certificate_hosts + + algorithm = "RSA" + rsa_bits = 2048 +} + +resource "tls_self_signed_cert" "this" { + for_each = local.certificate_hosts + + private_key_pem = tls_private_key.this[each.key].private_key_pem + + subject { + common_name = each.value + organization = "STACKIT Example" + } + + dns_names = [each.value] + validity_period_hours = 8760 + + allowed_uses = [ + "key_encipherment", + "digital_signature", + "server_auth", + ] +} + +resource "stackit_alb_certificate" "this" { + for_each = local.certificate_hosts + + project_id = var.stackit_project_id + region = var.stackit_region + name = "${var.name_prefix}-${each.key}" + private_key = tls_private_key.this[each.key].private_key_pem + public_key = tls_self_signed_cert.this[each.key].cert_pem +} diff --git a/examples/alb-multi-tenant-routing/070-alb.tf b/examples/alb-multi-tenant-routing/070-alb.tf new file mode 100644 index 0000000..f9d106e --- /dev/null +++ b/examples/alb-multi-tenant-routing/070-alb.tf @@ -0,0 +1,129 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "stackit_application_load_balancer" "this" { + project_id = var.stackit_project_id + region = var.stackit_region + name = "${var.name_prefix}-alb" + plan_id = var.alb_plan_id + labels = local.labels + + networks = [ + { + network_id = stackit_network.this.network_id + role = "ROLE_LISTENERS_AND_TARGETS" + } + ] + + listeners = [ + { + name = "https" + port = 443 + protocol = "PROTOCOL_HTTPS" + + https = { + certificate_config = { + certificate_ids = [for cert in stackit_alb_certificate.this : cert.cert_id] + } + } + + http = { + hosts = [ + # Hosts are matched on the Host header. Within a host the rules are + # evaluated in the order they are listed and the first match wins, + # so specific rules come first and the catch-all comes last. A rule + # without a path matches every path. + { + host = local.app_host + rules = [ + # A legacy endpoint that still lives in the web application while + # the rest of /api has moved to the api pool. + { + target_pool = local.pool_names["web"] + path = { exact_match = "/api/v1/legacy" } + }, + { + target_pool = local.pool_names["api"] + path = { prefix = "/api" } + }, + # Canary release of the web application: the same URL, selected by + # a request header (automated clients) or a query parameter (links + # handed to testers). Listed after /api, so the API is not affected. + { + target_pool = local.pool_names["canary"] + headers = [{ name = "X-Canary", exact_match = "true" }] + }, + { + target_pool = local.pool_names["canary"] + query_parameters = [{ name = "preview", exact_match = "true" }] + }, + { + target_pool = local.pool_names["web"] + path = { prefix = "/ws" } + web_socket = true + }, + { + target_pool = local.pool_names["web"] + path = { prefix = "/" } + }, + ] + }, + { + host = local.admin_host + rules = [ + # Stateful application: the load balancer sets a cookie and keeps + # routing the client to the backend that answered first. + { + target_pool = local.pool_names["admin"] + path = { prefix = "/" } + cookie_persistence = { + name = "admin-session" + ttl = var.session_cookie_ttl + } + }, + ] + }, + ] + } + } + ] + + target_pools = [ + for name, app in local.applications : { + name = local.pool_names[name] + target_port = app.port + targets = [ + for key, nic in stackit_network_interface.backend : { + display_name = "${var.name_prefix}-backend-${key}" + ip = nic.ipv4 + } + ] + active_health_check = local.active_health_check + } + ] + + options = { + # The load balancer creates and releases its public IP itself. + ephemeral_address = true + private_network_only = false + access_control = { + allowed_source_ranges = var.alb_allowed_source_ranges + } + } + + # The targets are the addresses of the network interfaces, which exist before + # the servers do. Wait for the servers so that the load balancer does not + # start its health checks against ports nobody listens on yet. + depends_on = [stackit_server.backend] +} diff --git a/examples/alb-multi-tenant-routing/100-outputs.tf b/examples/alb-multi-tenant-routing/100-outputs.tf new file mode 100644 index 0000000..1927eeb --- /dev/null +++ b/examples/alb-multi-tenant-routing/100-outputs.tf @@ -0,0 +1,38 @@ +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "alb_external_address" { + description = "Public IPv4 address of the Application Load Balancer, allocated by the load balancer itself." + value = stackit_application_load_balancer.this.external_address +} + +output "app_host" { + description = "Hostname of the web, api and canary applications." + value = local.app_host +} + +output "admin_host" { + description = "Hostname of the admin application." + value = local.admin_host +} + +output "target_pools" { + description = "Target pool names and the backend port each of them forwards to." + value = { for name, app in local.applications : local.pool_names[name] => app.port } +} + +output "backend_private_ips" { + description = "Private IPv4 addresses of the backend VMs, keyed by backend index." + value = { for key, nic in stackit_network_interface.backend : key => nic.ipv4 } +} diff --git a/examples/alb-multi-tenant-routing/MAINTAINERS.md b/examples/alb-multi-tenant-routing/MAINTAINERS.md new file mode 100644 index 0000000..52c53ff --- /dev/null +++ b/examples/alb-multi-tenant-routing/MAINTAINERS.md @@ -0,0 +1,10 @@ +# Maintainers + +General maintainers: + +- Tim Reibe (Tim.Reibe@digits.schwarz) +- Florian Schmidl (florian_michael.schmidl_ext@external.digits.schwarz) + +This example is actively maintained. The owner is responsible for reviewing and updating dependencies and functionalities on a monthly basis. +For questions, issues, or feature requests, please email general maintainers. +Please include the BP name and version in your request. We will track your request as an issue. diff --git a/examples/alb-multi-tenant-routing/README.md b/examples/alb-multi-tenant-routing/README.md new file mode 100644 index 0000000..7f5cb25 --- /dev/null +++ b/examples/alb-multi-tenant-routing/README.md @@ -0,0 +1,298 @@ + + +# ALB Multi-Tenant Routing + +One STACKIT Application Load Balancer in front of several applications: two hostnames on a single HTTPS listener, path, header and query parameter rules, cookie persistence, WebSocket, and a target pool with its own health check per application. + +## Overview + +The other load balancer examples in this repository put one wildcard host in front of one target pool. That is enough for a single application, but the layer 7 features of the Application Load Balancer (ALB) are made for the case where one load balancer serves many tenants: applications or teams that share one public address and one listener and are told apart by host, path, header or query parameter. This example builds that case: + +- **Host routing.** `app.` and `admin.` share the listener and the public IP. The load balancer selects the certificate by SNI and the rules by the `Host` header. +- **Path routing.** Under the app host, `/api` goes to the api pool and everything else to the web pool. One exact path, `/api/v1/legacy`, stays with the web application although the rest of the API has moved, which is how a monolith is taken apart one endpoint at a time. +- **Header and query parameter routing.** `X-Canary: true` or `?preview=true` send the same URL to a canary pool, so a new version can be released next to the current one without touching DNS or the clients. +- **Cookie persistence** on the admin pool keeps a client on the backend it started with. +- **WebSocket** upgrades on `/ws`. +- **One target pool per application**, each with an active HTTP health check, so an application that fails on one VM is removed from its own pool only. + +Two backend VMs in different availability zones run the same small HTTP application on four ports, one per pool. Every pool contains both VMs, so two machines are enough to make every route highly available. Each response names the pool and the VM that served it, which makes every rule provable from the response body. See [Testing](#testing). + +## Architecture + +```mermaid +flowchart LR + client([Client]) + client -- "app.example.internal" --> listener + client -- "admin.example.internal" --> listener + subgraph project["STACKIT project · network 10.20.1.0/24"] + subgraph alb["Application Load Balancer"] + listener["HTTPS listener :443
two certificates, selected by SNI"] + listener -- "/api/v1/legacy · /ws · /" --> web["web pool → :8081"] + listener -- "/api" --> api["api pool → :8082"] + listener -- "X-Canary: true · ?preview=true" --> canary["canary pool → :8084"] + listener -- "admin host
cookie persistence" --> admin["admin pool → :8083"] + end + b1["Backend 01 · eu01-1
:8081 :8082 :8083 :8084"] + b2["Backend 02 · eu01-2
:8081 :8082 :8083 :8084"] + web & api & canary & admin --> b1 & b2 + end +``` + +## What gets created + +| Component | Resource | Purpose | +| ------------- | ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| Network | `stackit_network`, `stackit_security_group`, `stackit_security_group_rule` | Private network and a security group that opens the four application ports inside the network | +| Backends | `stackit_server`, `stackit_network_interface` (one per AZ) | Debian VMs, provisioned by cloud-init with the test application (`files/server.py`) on ports 8081 to 8084 | +| Certificates | `tls_private_key`, `tls_self_signed_cert`, `stackit_alb_certificate` (one per host) | Self-signed certificates for `app.` and `admin.` | +| Load balancer | `stackit_application_load_balancer` | HTTPS listener with two hosts and seven rules, four target pools with health checks, public IP | + +The load balancer allocates its public IP itself (`options.ephemeral_address = true`) and releases it on destroy, so no `stackit_public_ip` is reserved up front. The address is bound to the lifetime of the load balancer; where DNS records have to point at a stable address, reserve a `stackit_public_ip` and pass it as `external_address` instead, as [`alb-tls-examples/vm-alb-self-signed-cert`](../alb-tls-examples/vm-alb-self-signed-cert/README.md) does. No DNS zone is created either; the hostnames are mapped to the address on the client with `curl --resolve`. + +The backends need a security group of their own (`stackit_security_group.backend`): the load balancer attaches its target security group to the backend interfaces, and that group only permits traffic from the load balancer. Without an additional group with outbound rules the VMs cannot reach the metadata service and cloud-init never runs. + +## Routing rules + +The listener carries two hosts. Hosts are matched on the `Host` header, exact names before wildcards before `*`. Within a host the load balancer evaluates the rules in the order they are listed and the first match wins, so the specific rules are listed first and the catch-all last. A rule without a `path` matches every path. + +| # | Host | Match | Target pool | Purpose | +| --- | ---------------- | ------------------------------------------- | --------------- | ---------------------------------------------------------------------------------- | +| 1 | `app.` | `path.exact_match = "/api/v1/legacy"` | `alb-mt-web` | Legacy endpoint that the web application still serves | +| 2 | `app.` | `path.prefix = "/api"` | `alb-mt-api` | The API; listed before the canary rules so that a canary header does not affect it | +| 3 | `app.` | header `X-Canary: true` | `alb-mt-canary` | Canary release, selected by automated clients | +| 4 | `app.` | query parameter `preview=true` | `alb-mt-canary` | Canary release, selected by a link handed to testers | +| 5 | `app.` | `path.prefix = "/ws"`, `web_socket = true` | `alb-mt-web` | WebSocket endpoint of the web application | +| 6 | `app.` | `path.prefix = "/"` | `alb-mt-web` | Everything else on the app host | +| 7 | `admin.` | `path.prefix = "/"`, cookie `admin-session` | `alb-mt-admin` | Admin application with cookie persistence | + +Path prefixes match on segment boundaries: `/api` matches `/api` and `/api/users` but not `/apiary`. Header names are matched case-insensitively; header and query parameter values are exact, case-sensitive matches. + +Each target pool forwards to its own port on both backends and runs the same active health check (`/healthz`, every 5 s, two failures to leave and two successes to rejoin the pool): + +| Target pool | Backend port | Rules | +| --------------- | ------------ | ------- | +| `alb-mt-web` | 8081 | 1, 5, 6 | +| `alb-mt-api` | 8082 | 2 | +| `alb-mt-admin` | 8083 | 7 | +| `alb-mt-canary` | 8084 | 3, 4 | + +## Prerequisites + +| Tool | Version | +| ------------- | -------- | +| Terraform | >= 1.5.0 | +| curl, openssl | any | + +A STACKIT service account key with the `editor` role on the target project is required. The project needs quota for one load balancer, one public IP, two VMs and three security groups (one of the example, two that the load balancer creates for itself and its targets). + +## Usage + +### 1. Configure variables + +```bash +cp terraform.tfvars.example terraform.tfvars +# fill in stackit_project_id and stackit_service_account_key_path +``` + +All other variables have defaults, see [`020-variables.tf`](020-variables.tf). `domain` changes the hostnames, `alb_plan_id` the service plan (`stackit beta alb plans` lists the plans of your region; `p10` is the only one in `eu01` at the time of writing and carries this configuration). If the project belongs to a STACKIT Network Area, `network_cidr` must lie inside the network ranges of that area; otherwise the network creation fails with `the prefix is not part of the allowed network ranges`. + +### 2. Deploy + +```bash +terraform init +terraform apply +``` + +The apply takes ten to fifteen minutes, most of it for the load balancer, which is created after the backends. By the time it is ready the backends have finished cloud-init and passed the health checks. If a request is still answered with `503 no healthy upstream`, wait a minute and retry. + +### 3. Verify + +The certificates are self-signed and no DNS zone exists, so the hostnames are mapped to the load balancer address with `--resolve`. That sets both the `Host` header and the TLS SNI, which is exactly what the load balancer routes on; the wildcard form maps every hostname on port 443 to the same address: + +```bash +export ALB_IP=$(terraform output -raw alb_external_address) +export APP=$(terraform output -raw app_host) +export ADMIN=$(terraform output -raw admin_host) + +curl -sk --resolve "*:443:$ALB_IP" "https://$APP/" +curl -sk --resolve "*:443:$ALB_IP" "https://$ADMIN/" +``` + +Both requests return a JSON body with the pool and the backend that answered: + +``` +{"host": "app.example.internal", "path": "/", "query": "", "pool": "web", "backend": "alb-mt-backend-01", "port": 8081} +{"host": "admin.example.internal", "path": "/", "query": "", "pool": "admin", "backend": "alb-mt-backend-02", "port": 8083} +``` + +## Testing + +Every rule can be proven from the `pool` field of the response. The commands below assume the variables from [Verify](#3-verify); the pools listed are what the deployment described here returned. Repeat a command a few times to see both VMs of a pool appear in the `backend` field; the order is not a strict alternation. + +### Host and path routing + +| Command | Pool | Why | +| ------------------------------------------------------------------ | ------- | --------------------------------------------------- | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/"` | `web` | Rule 6, catch-all of the app host | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/index.html"` | `web` | Rule 6 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/api"` | `api` | Rule 2, prefix matches the segment itself | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/api/users"` | `api` | Rule 2 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/api/v1/legacy"` | `web` | Rule 1, exact match listed before the `/api` prefix | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/api/v1/legacy/"` | `api` | Not the exact path any more, rule 2 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/api/v1/legacy2"` | `api` | Not the exact path, rule 2 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/apiary"` | `web` | `/api` does not match `/apiary`, rule 6 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$ADMIN/"` | `admin` | Rule 7, other host | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$ADMIN/api/users"` | `admin` | The `/api` rule exists under the app host only | + +The whole matrix in one go: + +```bash +for url in "$APP/" "$APP/index.html" "$APP/api" "$APP/api/users" "$APP/api/v1/legacy" \ + "$APP/api/v1/legacy/" "$APP/api/v1/legacy2" "$APP/apiary" "$ADMIN/" "$ADMIN/api/users"; do + printf '%-40s ' "$url"; curl -sk --resolve "*:443:$ALB_IP" "https://$url" +done +``` + +A hostname that matches no host entry is answered by the load balancer itself with `404` and an empty body, and so is the bare IP address: the hosts are matched on the `Host` header, SNI only selects the certificate. + +```bash +curl -sk --resolve "*:443:$ALB_IP" -o /dev/null -w '%{http_code}\n' "https://other.example.internal/" # 404 +curl -sk -o /dev/null -w '%{http_code}\n' "https://$ALB_IP/" # 404 +curl -sk -H "Host: $APP" "https://$ALB_IP/" # pool web +``` + +Port 80 has no listener; plain `http://` connections time out. + +### Header and query parameter routing + +| Command | Pool | Why | +| --------------------------------------------------------------------------------- | -------- | ----------------------------------------------------- | +| `curl -sk --resolve "*:443:$ALB_IP" -H 'X-Canary: true' "https://$APP/"` | `canary` | Rule 3 | +| `curl -sk --resolve "*:443:$ALB_IP" -H 'X-Canary: true' "https://$APP/checkout"` | `canary` | Rule 3, any path | +| `curl -sk --resolve "*:443:$ALB_IP" -H 'X-Canary: false' "https://$APP/"` | `web` | Value is an exact match, rule 6 | +| `curl -sk --resolve "*:443:$ALB_IP" -H 'x-canary: true' "https://$APP/"` | `canary` | Header names are case-insensitive | +| `curl -sk --resolve "*:443:$ALB_IP" -H 'X-Canary: True' "https://$APP/"` | `web` | Header values are case-sensitive, rule 6 | +| `curl -sk --resolve "*:443:$ALB_IP" -H 'X-Canary: true' "https://$APP/api/users"` | `api` | Rule 2 is listed before rule 3; the API has no canary | +| `curl -sk --resolve "*:443:$ALB_IP" -H 'X-Canary: true' "https://$APP/ws"` | `canary` | Rule 3 has no path and is listed before rule 5 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/?preview=true"` | `canary` | Rule 4 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/pricing?utm=x&preview=true"` | `canary` | Rule 4, other parameters are ignored | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/?preview=false"` | `web` | Value is an exact match, rule 6 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/?preview"` | `web` | No value, rule 6 | +| `curl -sk --resolve "*:443:$ALB_IP" "https://$APP/api/users?preview=true"` | `api` | Rule 2 is listed before rule 4 | +| `curl -sk --resolve "*:443:$ALB_IP" -H 'X-Canary: true' "https://$ADMIN/"` | `admin` | The canary rules exist under the app host only | + +### Certificates and SNI + +Both certificates are attached to the same listener. The load balancer presents the one whose name matches the SNI of the handshake: + +```bash +openssl s_client -connect "$ALB_IP:443" -servername "$APP" /dev/null | openssl x509 -noout -subject -nameopt RFC2253 +openssl s_client -connect "$ALB_IP:443" -servername "$ADMIN" /dev/null | openssl x509 -noout -subject -nameopt RFC2253 +openssl s_client -connect "$ALB_IP:443" -noservername /dev/null | openssl x509 -noout -subject -nameopt RFC2253 +``` + +``` +subject=CN=app.example.internal,O=STACKIT Example +subject=CN=admin.example.internal,O=STACKIT Example +subject=CN=admin.example.internal,O=STACKIT Example +``` + +A handshake without SNI, or with a name that matches none of the certificates, gets one of the attached certificates (the admin certificate in this deployment). Clients have to send the hostname as SNI to receive the matching certificate, which every browser and `curl --resolve` do. + +### Cookie persistence + +Without the cookie the requests to the admin pool are spread over both backends. The first response sets `admin-session`; requests that send it back land on the same backend: + +```bash +# no cookie: both backends answer +for i in 1 2 3 4 5 6; do curl -sk --resolve "*:443:$ALB_IP" "https://$ADMIN/" | grep -o '"backend": "[^"]*"'; done | sort | uniq -c + +# the first response sets the cookie ... +curl -sk --resolve "*:443:$ALB_IP" -c cookies.txt -D - -o /dev/null "https://$ADMIN/" | grep -i set-cookie + +# ... and every request that sends it back reaches the same backend +for i in 1 2 3 4 5 6 7 8; do curl -sk --resolve "*:443:$ALB_IP" -b cookies.txt "https://$ADMIN/" | grep -o '"backend": "[^"]*"'; done | sort | uniq -c +``` + +``` + 4 "backend": "alb-mt-backend-01" + 2 "backend": "alb-mt-backend-02" +set-cookie: admin-session="910f5fe4cd453715"; Max-Age=300; HttpOnly + 8 "backend": "alb-mt-backend-02" +``` + +`Max-Age` is the `session_cookie_ttl` variable. Responses to requests that already carry the cookie do not set it again, and the app host has no cookie persistence: `curl -sk --resolve "*:443:$ALB_IP" -D - -o /dev/null "https://$APP/"` returns no `Set-Cookie` header. + +### WebSocket + +Rule 5 enables WebSocket for `/ws`. A handshake request is answered with `101 Switching Protocols`, after which the backend sends one text frame that names pool and backend and echoes every frame it receives. The upgrade is an HTTP/1.1 mechanism, so curl is pinned to HTTP/1.1; it prints the response headers and the raw frame and gives up after the timeout, which is expected: + +```bash +curl -sk --resolve "*:443:$ALB_IP" --http1.1 -i -N -m 3 \ + -H 'Connection: Upgrade' -H 'Upgrade: websocket' \ + -H 'Sec-WebSocket-Version: 13' -H "Sec-WebSocket-Key: $(openssl rand -base64 16)" \ + "https://$APP/ws" +``` + +``` +HTTP/1.1 101 Switching Protocols +server: envoy +upgrade: websocket +connection: Upgrade +sec-websocket-accept: mFhn+IQx4r4Tgn7ryTMikIcIhn0= + +{"pool": "web", "backend": "alb-mt-backend-01", "port": 8081, "message": "websocket established"} +``` + +The `sec-websocket-accept` value is derived from the key and differs per run; the two bytes in front of the JSON are the frame header. The same request to `/` (rule 6, no `web_socket`) is rejected by the load balancer with `403 Forbidden` and `connection: close`; the upgrade never reaches the backend. + +### Health checks per pool + +Each pool has its own health check, so an application that fails on one VM leaves its own pool only. `/healthz/fail/` marks the pool that answers as unhealthy on the VM that answers for that many seconds (at most 3600; `/healthz/fail` uses 300). The path is matched by its last segments, so it works under every rule: + +```bash +# mark the web application on the backend that answers as unhealthy for 60 s +curl -sk --resolve "*:443:$ALB_IP" "https://$APP/healthz/fail/60" + +# after about ten seconds (two failed checks) every request to the web pool +# reaches the other backend ... +sleep 15 +for i in 1 2 3 4 5 6 7 8; do curl -sk --resolve "*:443:$ALB_IP" "https://$APP/" | grep -o '"backend": "[^"]*"'; done | sort | uniq -c + +# ... while the api pool on the same VM is still healthy and still uses both backends +for i in 1 2 3 4 5 6 7 8; do curl -sk --resolve "*:443:$ALB_IP" "https://$APP/api/users" | grep -o '"backend": "[^"]*"'; done | sort | uniq -c +``` + +``` +{"healthy": false, "unhealthy_for_s": 60, "pool": "web", "backend": "alb-mt-backend-02", "port": 8081} + 8 "backend": "alb-mt-backend-01" + 2 "backend": "alb-mt-backend-01" + 6 "backend": "alb-mt-backend-02" +``` + +The backend rejoins the pool on its own once the duration has passed and the check has succeeded twice. A backend that is unhealthy receives no traffic, so `/healthz/ok`, which clears the state, only reaches a backend that is still or again in the pool. Marking both backends of a pool unhealthy makes the load balancer answer `503 no healthy upstream` for that pool while the other pools keep working. + +## Notes + +- Rule order is significant. The load balancer takes the first matching rule, so a catch-all `prefix = "/"` listed first shadows every rule after it. Keep exact matches and long prefixes before short ones and put header and query parameter rules where their scope should start. +- A rule without `path` matches every path. Rules 3 and 4 therefore also cover `/ws`, and neither has `web_socket`, so a canary client cannot open a WebSocket; a canary rule for `/ws` with `web_socket = true` listed before rule 3 would allow it. +- Requests whose `Host` matches no host entry are answered by the load balancer with an empty `404`; no backend is contacted. A host entry `*` gives such requests a default. +- Header names are matched case-insensitively, header and query parameter values case-sensitively and in full. `X-Canary: True` does not match `exact_match = "true"`. +- The private keys of the certificates are stored in the Terraform state. Protect the state accordingly. +- `stackit beta alb plans` lists the service plans of your region. At the time of writing only `p10` (two VMs in an active/passive setup, 10,000 connections) exists in `eu01`; further plans are announced. The limits of the resource are 20 listeners, 20 target pools and 250 targets per pool. + +## Cleanup + +```bash +terraform destroy +``` + +The load balancer releases its ephemeral public IP on destroy. + +## References + +- [Application Load Balancer: basic concepts](https://docs.stackit.cloud/products/network/load-balancing-and-content-delivery/application-load-balancer/basics/basic-concepts-alb/) +- [Application Load Balancer: features](https://docs.stackit.cloud/products/network/load-balancing-and-content-delivery/application-load-balancer/basics/features-alb/) +- [Application Load Balancer: service plans](https://docs.stackit.cloud/products/network/load-balancing-and-content-delivery/application-load-balancer/reference/service-plans/) +- [Terraform provider: `stackit_application_load_balancer`](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/application_load_balancer) +- [Terraform provider: `stackit_alb_certificate`](https://registry.terraform.io/providers/stackitcloud/stackit/latest/docs/resources/alb_certificate) diff --git a/examples/alb-multi-tenant-routing/cloud-init.yaml.tftpl b/examples/alb-multi-tenant-routing/cloud-init.yaml.tftpl new file mode 100644 index 0000000..16a8fa4 --- /dev/null +++ b/examples/alb-multi-tenant-routing/cloud-init.yaml.tftpl @@ -0,0 +1,25 @@ +#cloud-config +write_files: + - path: /opt/backend/server.py + permissions: "0755" + content: | + ${indent(6, chomp(server_py))} + - path: /etc/systemd/system/backend.service + permissions: "0644" + content: | + [Unit] + Description=HTTP backend for load balancer routing tests + After=network-online.target + Wants=network-online.target + + [Service] + ExecStart=/usr/bin/python3 /opt/backend/server.py ${backend_arguments} + Restart=always + RestartSec=2 + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl daemon-reload + - systemctl enable --now backend.service diff --git a/examples/alb-multi-tenant-routing/files/server.py b/examples/alb-multi-tenant-routing/files/server.py new file mode 100644 index 0000000..77f2a74 --- /dev/null +++ b/examples/alb-multi-tenant-routing/files/server.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# Copyright 2026 Schwarz Digits Cloud GmbH & Co. KG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# HTTP backend for load balancer routing tests. The same application listens +# on one port per target pool and every response names the pool and the VM +# that served it. Each pool can be marked unhealthy on purpose and the +# application answers WebSocket upgrades with an echo service. +# Usage: server.py = [= ...] +import base64 +import hashlib +import json +import socket +import struct +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + +# port (as string) -> pool name +APPLICATIONS = dict(arg.split("=", 1) for arg in sys.argv[1:]) or {"8080": "default"} +HOSTNAME = socket.gethostname() +FAIL_SECONDS = 300 +WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" +LOCK = threading.Lock() +UNHEALTHY_UNTIL = {port: 0.0 for port in APPLICATIONS} + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): + # Silence the per-request log of BaseHTTPRequestHandler; the health + # checks of the load balancer alone would add a line every few seconds. + pass + + @property + def port(self): + return str(self.server.server_address[1]) + + @property + def pool(self): + return APPLICATIONS[self.port] + + def reply(self, status, body): + body.update({"pool": self.pool, "backend": HOSTNAME, "port": int(self.port)}) + payload = (json.dumps(body) + "\n").encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(payload) + + def do_GET(self): + url = urlsplit(self.path) + parts = [part for part in url.path.split("/") if part] + if "healthz" in parts: + return self.health(parts[parts.index("healthz") + 1 :]) + if self.headers.get("Upgrade", "").lower() == "websocket": + return self.websocket() + self.reply( + 200, + {"host": self.headers.get("Host"), "path": url.path, "query": url.query}, + ) + + do_HEAD = do_GET + + def health(self, action): + # /healthz reports the state, /healthz/fail[/] marks this + # pool on this VM unhealthy (default 300 s), /healthz/ok recovers it. + now = time.time() + with LOCK: + if action[:1] == ["fail"]: + seconds = FAIL_SECONDS + if len(action) > 1 and action[1].isdecimal(): + seconds = min(int(action[1]), 3600) + UNHEALTHY_UNTIL[self.port] = now + seconds + elif action[:1] == ["ok"]: + UNHEALTHY_UNTIL[self.port] = 0.0 + remaining = max(0, int(UNHEALTHY_UNTIL[self.port] - now)) + healthy = remaining == 0 + status = 200 if healthy or action else 503 + self.reply(status, {"healthy": healthy, "unhealthy_for_s": remaining}) + + def websocket(self): + key = self.headers.get("Sec-WebSocket-Key", "") + accept = base64.b64encode( + hashlib.sha1((key + WS_GUID).encode()).digest() + ).decode() + self.send_response(101) + self.send_header("Upgrade", "websocket") + self.send_header("Connection", "Upgrade") + self.send_header("Sec-WebSocket-Accept", accept) + self.end_headers() + self.close_connection = True + greeting = { + "pool": self.pool, + "backend": HOSTNAME, + "port": int(self.port), + "message": "websocket established", + } + self.send_frame(0x1, json.dumps(greeting).encode()) + while True: + header = self.rfile.read(2) + if len(header) < 2: + return + fin = header[0] & 0x80 + opcode = header[0] & 0x0F + length = header[1] & 0x7F + if length == 126: + length = struct.unpack("!H", self.rfile.read(2))[0] + elif length == 127: + length = struct.unpack("!Q", self.rfile.read(8))[0] + if length > 65536: + self.send_frame(0x8, struct.pack("!H", 1009)) + return + mask = self.rfile.read(4) if header[1] & 0x80 else bytes(4) + payload = bytes( + b ^ mask[i % 4] for i, b in enumerate(self.rfile.read(length)) + ) + if opcode == 0x8: + self.send_frame(0x8, payload[:2]) + return + if opcode == 0x9: + self.send_frame(0xA, payload) + elif opcode in (0x0, 0x1, 0x2): + self.send_frame(opcode, payload, fin) + + def send_frame(self, opcode, payload, fin=0x80): + header = bytes([fin | opcode]) + if len(payload) < 126: + header += bytes([len(payload)]) + elif len(payload) < 65536: + header += bytes([126]) + struct.pack("!H", len(payload)) + else: + header += bytes([127]) + struct.pack("!Q", len(payload)) + self.wfile.write(header + payload) + self.wfile.flush() + + +def serve(port): + ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() + + +if __name__ == "__main__": + threads = [ + threading.Thread(target=serve, args=(int(port),), daemon=True) + for port in APPLICATIONS + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() diff --git a/examples/alb-multi-tenant-routing/terraform.tfvars.example b/examples/alb-multi-tenant-routing/terraform.tfvars.example new file mode 100644 index 0000000..7374ac2 --- /dev/null +++ b/examples/alb-multi-tenant-routing/terraform.tfvars.example @@ -0,0 +1,8 @@ +# Copy this file to terraform.tfvars and fill in your values. +# stackit_region defaults to "eu01" and can be omitted. + +stackit_project_id = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" +stackit_service_account_key_path = "/path/to/stackit-sa.json" + +# Optional: hostnames become app. and admin.. +# domain = "example.internal"