Implement multinode testing for Rocky 10

.github/workflows/multinode-inputs.py

@@ -31,12 +31,14 @@ class Scenario:
 
 
 ROCKY_9 = OSRelease("rocky", "9", "cloud-user")
+ROCKY_10 = OSRelease("rocky", "10", "cloud-user")
 UBUNTU_JAMMY = OSRelease("ubuntu", "jammy", "ubuntu")
 UBUNTU_NOBLE = OSRelease("ubuntu", "noble", "ubuntu")
 # NOTE(upgrade): Add supported releases here.
 OPENSTACK_RELEASES = [
     OpenStackRelease("2024.1", "2023.1", [ROCKY_9, UBUNTU_JAMMY]),
     OpenStackRelease("2025.1", "2024.1", [ROCKY_9, UBUNTU_NOBLE]),
+    OpenStackRelease("2025.1", "", [ROCKY_10]),
 ]
 NEUTRON_PLUGINS = ["ovs", "ovn"]
 VERSION_HIERARCHY = ["2023.1", "2024.1", "2025.1"]
@@ -75,7 +77,7 @@ def random_scenario() -> Scenario:
     openstack_release = random.choice(OPENSTACK_RELEASES)
     os_release = random.choice(openstack_release.os_releases)
     neutron_plugin = random.choice(NEUTRON_PLUGINS)
-    upgrade = "major" if random.random() > 0.6 else "none"
+    upgrade = "major" if (random.random() > 0.6 and openstack_release.previous_version not "") else "none"
     return Scenario(openstack_release, os_release, neutron_plugin, upgrade)
 
 

.github/workflows/stackhpc-multinode.yml

@@ -14,9 +14,10 @@ name: Multinode
       os_distribution:
         description: Host OS distribution
         type: choice
-        default: rocky
+        default: rocky-9
         options:
-          - rocky
+          - rocky-9
+          - rocky-10
           - ubuntu
       neutron_plugin:
         description: Neutron ML2 plugin
@@ -95,12 +96,12 @@ jobs:
   multinode:
     name: Multinode
     needs: github_user_ssh_keys
-    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@1.8.0
+    uses: stackhpc/stackhpc-openstack-gh-workflows/.github/workflows/multinode.yml@main
     with:
       multinode_name: ${{ inputs.multinode_name }}
-      os_distribution: ${{ inputs.os_distribution }}
-      os_release: ${{ inputs.os_distribution == 'rocky' && '9' || 'noble' }}
-      ssh_username: ${{ inputs.os_distribution == 'rocky' && 'cloud-user' || 'ubuntu' }}
+      os_distribution: ${{ inputs.os_distribution == 'ubuntu' && 'ubuntu' || 'rocky' }}
+      os_release: ${{ inputs.os_distribution == 'rocky-9' && '9' || inputs.os_distribution == 'rocky-10' && '10' || 'noble' }}
+      ssh_username: ${{ (inputs.os_distribution == 'rocky-9' || inputs.os_distribution == 'rocky-10') && 'cloud-user' || 'ubuntu' }}
       neutron_plugin: ${{ inputs.neutron_plugin }}
       upgrade: ${{ inputs.upgrade }}
       break_on: ${{ inputs.break_on }}

etc/kayobe/ansible/ceph/cephadm-deploy.yml

@@ -7,6 +7,28 @@
     - cephadm
     - cephadm-deploy
   tasks:
+    # note(owenjones): the Ceph release signing key still uses SHA-1, which requires us to
+    # downgrade to the LEGACY policy on Rocky 10 until a new key is published.
+    # A later Ceph release might fix this - https://tracker.ceph.com/issues/74861
+    - name: Discover currently active crypto policy (Rocky 10)
+      ansible.builtin.command:
+        cmd: update-crypto-policies --show
+      become: true
+      register: crypto_policy
+      when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '10'
+    - name: Set crypto policy to LEGACY (Rocky 10)
+      ansible.builtin.command:
+        cmd: update-crypto-policies --set LEGACY
+      become: true
+      when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '10'
     - name: Apply Cephadm role
-      ansible.builtin.import_role:
-        name: stackhpc.cephadm.cephadm
+      block:
+        - name: Apply Cephadm role
+          ansible.builtin.import_role:
+            name: stackhpc.cephadm.cephadm
+      always:
+        - name: Revert crypto policy to {{ crypto_policy.stdout }} (Rocky 10)
+          ansible.builtin.command:
+            cmd: update-crypto-policies --set {{ crypto_policy.stdout }}
+          become: true
+          when: ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '10'

etc/kayobe/ansible/requirements.yml

@@ -1,7 +1,7 @@
 ---
 collections:
   - name: stackhpc.cephadm
-    version: 1.22.0
+    version: 1.22.1
   - name: pulp.squeezer
     version: 0.3.0
   - name: stackhpc.pulp

etc/kayobe/environments/ci-multinode/kolla.yml

@@ -1,12 +1,13 @@
 ---
-kolla_enable_cinder: true
+kolla_enable_barbican: true
 kolla_enable_cinder_backup: true
+kolla_enable_cinder: true
+kolla_enable_designate: true
+kolla_enable_magnum: true
 kolla_enable_neutron_provider_networks: true
 kolla_enable_octavia: true
-kolla_enable_magnum: true
-kolla_enable_designate: true
-kolla_enable_redis: true
-kolla_enable_barbican: true
+kolla_enable_redis: "{{ kolla_base_distro_and_version in ['ubuntu-noble', 'rocky-9'] }}"
+kolla_enable_valkey: "{{ kolla_base_distro_and_version == 'rocky-10' }}"
 
 kolla_enable_tls_external: true
 kolla_enable_tls_internal: true

etc/kayobe/environments/ci-multinode/stackhpc-ci.yml

@@ -22,36 +22,52 @@ stackhpc_repo_mirror_password: !vault |
   33306539316466373763656137653232313732646533353333626665313936613736
 
 # Build and deploy released Pulp repository versions.
-stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
-stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}"
-stackhpc_repo_rhel9_rabbitmq_erlang_26_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_26_version }}"
-stackhpc_repo_rhel9_rabbitmq_erlang_27_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_27_version }}"
-stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}"
-stackhpc_repo_ubuntu_noble_version: "{{ stackhpc_pulp_repo_ubuntu_noble_version }}"
-stackhpc_repo_ubuntu_noble_security_version: "{{ stackhpc_pulp_repo_ubuntu_noble_security_version }}"
-stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
-stackhpc_repo_docker_ce_ubuntu_noble_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_noble_version }}"
+# Build against released Pulp repository versions.
+# Minor versions for stackhpc_repo_rocky_X_* repos are derived from
+# stackhpc_pulp_repo_rocky_X_minor_version in etc/kayobe/pulp.yml
+stackhpc_repo_almalinux_9_proxysql_2_7_version: "{{ stackhpc_pulp_repo_almalinux_9_proxysql_2_7_version }}"
+stackhpc_repo_centos_stream_10_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_10_docker_version }}"
+stackhpc_repo_centos_stream_10_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_10_nfv_openvswitch_version }}"
+stackhpc_repo_centos_stream_10_openstack_epoxy_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_10_openstack_epoxy_version }}"
+stackhpc_repo_centos_stream_10_ovn_lts_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_10_ovn_lts_version }}"
+stackhpc_repo_centos_stream_10_storage_ceph_squid_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_10_storage_ceph_squid_version }}"
+stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}"
 stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_nfv_openvswitch_version }}"
 stackhpc_repo_centos_stream_9_openstack_epoxy_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_openstack_epoxy_version }}"
 stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_opstools_version }}"
 stackhpc_repo_centos_stream_9_storage_ceph_squid_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_storage_ceph_squid_version }}"
-stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_multiarch_centos_stream_9_docker_version }}"
-stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}"
-stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}"
-stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}"
+stackhpc_repo_docker_ce_ubuntu_noble_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_noble_version }}"
+stackhpc_repo_epel_10_version: "{{ stackhpc_pulp_repo_multiarch_epel_10_version }}"
 stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_multiarch_epel_9_version }}"
+stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
 stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}"
 stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}"
-stackhpc_repo_almalinux_9_proxysql_2_7_version: "{{ stackhpc_pulp_repo_almalinux_9_proxysql_2_7_version }}"
-## Use derived vars from etc/kayobe/pulp.yml to switch between
-## minor Rocky versions using stackhpc_pulp_repo_rocky_x_minor_version
-stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_pulp_repo_rocky_9_baseos_version }}"
+stackhpc_repo_rhel_10_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_10_mariadb_10_11_version }}"
+stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_influxdb_version }}"
+stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_mariadb_10_11_version }}"
+stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_multiarch_rhel_9_treasuredata_5_version }}"
+stackhpc_repo_rhel10_doca_version: "{{ stackhpc_pulp_repo_rhel10_doca_version }}"
+stackhpc_repo_rhel9_doca_version: "{{ stackhpc_pulp_repo_rhel9_doca_version }}"
+stackhpc_repo_rhel9_rabbitmq_erlang_26_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_26_version }}"
+stackhpc_repo_rhel9_rabbitmq_erlang_27_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_27_version }}"
+stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_multiarch_rhel9_rabbitmq_erlang_version }}"
+stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}"
+stackhpc_repo_rocky_10_appstream_version: "{{ stackhpc_pulp_repo_rocky_10_appstream_version }}"
+stackhpc_repo_rocky_10_baseos_version: "{{ stackhpc_pulp_repo_rocky_10_baseos_version }}"
+stackhpc_repo_rocky_10_crb_version: "{{ stackhpc_pulp_repo_rocky_10_crb_version }}"
+stackhpc_repo_rocky_10_extras_version: "{{ stackhpc_pulp_repo_rocky_10_extras_version }}"
+stackhpc_repo_rocky_10_highavailability_version: "{{ stackhpc_pulp_repo_rocky_10_highavailability_version }}"
+stackhpc_repo_rocky_10_security_version: "{{ stackhpc_pulp_repo_rocky_10_security_version }}"
 stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstream_version }}"
-stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}"
+stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_pulp_repo_rocky_9_baseos_version }}"
 stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
+stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}"
 stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
 stackhpc_repo_rocky_9_security_version: "{{ stackhpc_pulp_repo_rocky_9_security_version }}"
 stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_multiarch_rocky_9_sig_security_common_version }}"
+stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
+stackhpc_repo_ubuntu_noble_security_version: "{{ stackhpc_pulp_repo_ubuntu_noble_security_version }}"
+stackhpc_repo_ubuntu_noble_version: "{{ stackhpc_pulp_repo_ubuntu_noble_version }}"
 
 # Rocky-and-CI-specific Pulp urls
 stackhpc_include_os_minor_version_in_repo_url: true

etc/kayobe/kolla-image-tags.yml

@@ -5,12 +5,11 @@
 kolla_image_tags:
   openstack:
     rocky-9: 2025.1-rocky-9-20260603T214242
-    rocky-10: 2025.1-rocky-10-20260603T214242
+    rocky-10: 2025.1-rocky-10-20260612T161908
     ubuntu-noble: 2025.1-ubuntu-noble-20260603T214242
   bifrost:
     rocky-9: 2025.1-rocky-9-20260205T152450
     rocky-10: 2025.1-rocky-10-20260423T154048
     ubuntu-noble: 2025.1-ubuntu-noble-20260205T152450
   ovn:
     rocky-9: 2025.1-rocky-9-20260608T100935
-    rocky-10: 2025.1-rocky-10-20260609T095958

etc/kayobe/pulp-host-image-versions.yml

@@ -3,6 +3,6 @@
 # These images must be in SMS, since they are used by our AIO CI runners
 stackhpc_rocky_9_overcloud_host_image_version: 2025.1-20260520T124124
 stackhpc_rocky_9_overcloud_host_image_version_aarch64: 2025.1-20260420T162634
-stackhpc_rocky_10_overcloud_host_image_version: 2025.1-20260420T162634
-stackhpc_rocky_10_overcloud_host_image_version_aarch64: 2025.1-20260420T162634
+stackhpc_rocky_10_overcloud_host_image_version: 2025.1-20260612T162430
+stackhpc_rocky_10_overcloud_host_image_version_aarch64: 2025.1-20260612T162430
 stackhpc_ubuntu_noble_overcloud_host_image_version: 2025.1-20260420T162634

etc/kayobe/pulp-repo-versions.yml

@@ -76,6 +76,24 @@ stackhpc_pulp_repo_rocky_10_1_highavailability_version: 20260506T223941
 stackhpc_pulp_repo_rocky_10_1_security_aarch64_version: 20260510T225242
 stackhpc_pulp_repo_rocky_10_1_security_source_version: 20260510T222658
 stackhpc_pulp_repo_rocky_10_1_security_version: 20260510T223302
+stackhpc_pulp_repo_rocky_10_2_appstream_aarch64_version: 20260602T012631
+stackhpc_pulp_repo_rocky_10_2_appstream_source_version: 20260602T004048
+stackhpc_pulp_repo_rocky_10_2_appstream_version: 20260602T001113
+stackhpc_pulp_repo_rocky_10_2_baseos_aarch64_version: 20260602T012631
+stackhpc_pulp_repo_rocky_10_2_baseos_source_version: 20260602T004048
+stackhpc_pulp_repo_rocky_10_2_baseos_version: 20260602T011716
+stackhpc_pulp_repo_rocky_10_2_crb_aarch64_version: 20260602T012631
+stackhpc_pulp_repo_rocky_10_2_crb_source_version: 20260602T004048
+stackhpc_pulp_repo_rocky_10_2_crb_version: 20260602T001113
+stackhpc_pulp_repo_rocky_10_2_extras_aarch64_version: 20260602T012631
+stackhpc_pulp_repo_rocky_10_2_extras_source_version: 20260602T004048
+stackhpc_pulp_repo_rocky_10_2_extras_version: 20260602T001113
+stackhpc_pulp_repo_rocky_10_2_highavailability_aarch64_version: 20260602T012631
+stackhpc_pulp_repo_rocky_10_2_highavailability_source_version: 20260602T004048
+stackhpc_pulp_repo_rocky_10_2_highavailability_version: 20260602T001113
+stackhpc_pulp_repo_rocky_10_2_security_aarch64_version: 20260602T012631
+stackhpc_pulp_repo_rocky_10_2_security_source_version: 20260602T004048
+stackhpc_pulp_repo_rocky_10_2_security_version: 20260602T001113
 stackhpc_pulp_repo_rocky_9_1_appstream_version: 20231207T013715
 stackhpc_pulp_repo_rocky_9_1_baseos_version: 20231206T014015
 stackhpc_pulp_repo_rocky_9_1_crb_version: 20231211T120328

etc/kayobe/pulp.yml

@@ -267,8 +267,8 @@ stackhpc_pulp_rpm_architectures:
 
 # Whether to sync Rocky Linux 10 packages.
 stackhpc_pulp_sync_rocky_10: "{{ os_distribution == 'rocky' and os_release == '10' }}"
-# Rocky 10 minor version number. Supported values: 1. Default is 1
-stackhpc_pulp_repo_rocky_10_minor_version: '1'
+# Rocky 10 minor version number. Supported values: 1, 2. Default is 2
+stackhpc_pulp_repo_rocky_10_minor_version: '2'
 # Rocky 10 Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml for the selected minor release.
 

etc/kayobe/stackhpc-overcloud-dib.yml

@@ -37,7 +37,7 @@ stackhpc_overcloud_dib_elements:
 # StackHPC overcloud DIB image environment variables.
 stackhpc_overcloud_dib_env_vars: "{{ stackhpc_overcloud_dib_env_vars_default | combine(stackhpc_overcloud_dib_env_vars_ark if stackhpc_build_overcloud_image_from_pulp_package_mirrors | bool else {}) }}"
 
-# By default the :9 tag is used as base image. Here we want a specific minor version.
+# By default the :9/:10 tag is used as base image. Here we want a specific minor version.
 stackhpc_overcloud_dib_container_opts_default: >-
     {% if os_distribution == 'rocky' and os_release == '9' %}
       --build-arg=ROCKY_VERSION=9.{{ stackhpc_pulp_repo_rocky_9_minor_version }}

releasenotes/notes/rocky-10-multinode-cbcb0f38859626e7.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    cephadm deployment now supported in Rocky 10; The cephadm collection has
+    been bumped to 1.22.1 which updates the Ceph repository URL for Squid on
+    Rocky 10. The Rocky crypto policy in use during Ceph deployment is required
+    to be downgraded to LEGACY in order to use Ceph's release signing key. The
+    policy is reverted after deployment or on deplyment failure.

releasenotes/notes/rocky-10.2-761fd51ca5aefc88.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Repositories for Rocky 10.2 have been added, and new host and container images
+    have been built using them.