elasticsearch: fix keystore restart notify and non-rolling upgrade

roles/elasticsearch/tasks/elasticsearch-keystore.yml

@@ -19,7 +19,7 @@
       - bootstrap.password
     stdin: "{{ elasticsearch_bootstrap_pw }}"
   when: "'bootstrap.password' not in elasticsearch_keystore.stdout_lines"
-  changed_when: false
+  changed_when: true
   no_log: true
   notify:
     - Restart Elasticsearch
@@ -48,7 +48,7 @@
       - -x
       - xpack.security.http.ssl.keystore.secure_password
     stdin: "{{ elasticsearch_tls_key_passphrase }}"
-  changed_when: false
+  changed_when: true
   no_log: true
   when:
     - elasticsearch_http_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_keystore_secure_password.stdout
@@ -62,7 +62,7 @@
       - /usr/share/elasticsearch/bin/elasticsearch-keystore
       - remove
       - xpack.security.http.ssl.keystore.secure_password
-  changed_when: false
+  changed_when: true
   no_log: true
   when:
     - "'xpack.security.http.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines"
@@ -93,7 +93,7 @@
       - -x
       - xpack.security.http.ssl.truststore.secure_password
     stdin: "{{ elasticsearch_tls_key_passphrase }}"
-  changed_when: false
+  changed_when: true
   no_log: true
   when:
     - elasticsearch_http_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_http_ssl_truststore_secure_password.stdout
@@ -107,7 +107,7 @@
       - /usr/share/elasticsearch/bin/elasticsearch-keystore
       - remove
       - xpack.security.http.ssl.truststore.secure_password
-  changed_when: false
+  changed_when: true
   no_log: true
   when:
     - "'xpack.security.http.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines"
@@ -138,7 +138,7 @@
       - -x
       - xpack.security.transport.ssl.keystore.secure_password
     stdin: "{{ elasticsearch_tls_key_passphrase }}"
-  changed_when: false
+  changed_when: true
   no_log: true
   when:
     - elasticsearch_transport_ssl_keystore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_keystore_secure_password.stdout
@@ -152,7 +152,7 @@
       - /usr/share/elasticsearch/bin/elasticsearch-keystore
       - remove
       - xpack.security.transport.ssl.keystore.secure_password
-  changed_when: false
+  changed_when: true
   no_log: true
   when:
     - "'xpack.security.transport.ssl.keystore.secure_password' in elasticsearch_keystore.stdout_lines"
@@ -183,7 +183,7 @@
       - -x
       - xpack.security.transport.ssl.truststore.secure_password
     stdin: "{{ elasticsearch_tls_key_passphrase }}"
-  changed_when: false
+  changed_when: true
   no_log: true
   when:
     - elasticsearch_transport_ssl_truststore_secure_password.stdout is undefined or elasticsearch_tls_key_passphrase != elasticsearch_transport_ssl_truststore_secure_password.stdout
@@ -197,7 +197,7 @@
       - /usr/share/elasticsearch/bin/elasticsearch-keystore
       - remove
       - xpack.security.transport.ssl.truststore.secure_password
-  changed_when: false
+  changed_when: true
   no_log: true
   when:
     - "'xpack.security.transport.ssl.truststore.secure_password' in elasticsearch_keystore.stdout_lines"

roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml

@@ -73,14 +73,14 @@
 
     # Usually we should not need this step. It's only there to recover from broken upgrade plays
     # Without this step the cluster would never recover and the play would always fail
     - name: Enable shard allocation for the cluster
       ansible.builtin.uri:
         url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings"
         method: PUT
         body: '{ "persistent": { "cluster.routing.allocation.enable": null }}'
         body_format: json
         user: elastic
-        password: "{{ elasticstack_password.stdout }}"
+        password: "{{ __elasticstack_password }}"
         validate_certs: no
       register: response
       # next line is boolean not string, so no quotes around true
@@ -91,12 +91,12 @@
 
       # this step is key!!!  Don't restart more nodes
       # until all shards have completed recovery
     - name: Wait for cluster health to return to green
       ansible.builtin.uri:
         url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/health"
         method: GET
         user: elastic
-        password: "{{ elasticstack_password.stdout }}"
+        password: "{{ __elasticstack_password }}"
         validate_certs: no
       register: response
       until: "response.json.status == 'green'"
@@ -111,15 +111,15 @@
         body: '{ "persistent": { "cluster.routing.allocation.enable": "none" }}'
         body_format: json
         user: elastic
-        password: "{{ elasticstack_password.stdout }}"
+        password: "{{ __elasticstack_password }}"
         validate_certs: no
 
     - name: Stop non essential indexing to speed up shard recovery
       ansible.builtin.uri:
         url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_flush"
         method: POST
         user: elastic
-        password: "{{ elasticstack_password.stdout }}"
+        password: "{{ __elasticstack_password }}"
         validate_certs: no
       failed_when: false
 
@@ -171,12 +171,12 @@
         port: "{{ elasticstack_elasticsearch_http_port }}"
         delay: 30
 
     - name: Confirm the node joins the cluster # noqa: risky-shell-pipe
       ansible.builtin.shell: >
         if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
         curl
         -k
-        -u elastic:{{ elasticstack_password.stdout }}
+        -u elastic:{{ __elasticstack_password }}
         -s
         -m 2
         '{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes?h=name'
@@ -189,14 +189,14 @@
       delay: 3
       changed_when: false
 
     - name: Enable shard allocation for the cluster
       ansible.builtin.uri:
         url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/settings"
         method: PUT
         body: '{ "persistent": { "cluster.routing.allocation.enable": null }}'
         body_format: json
         user: elastic
-        password: "{{ elasticstack_password.stdout }}"
+        password: "{{ __elasticstack_password }}"
         validate_certs: no
       register: response
       # next line is boolean not string, so no quotes around true
@@ -205,12 +205,12 @@
       retries: 5
       delay: 30
 
     - name: Wait for cluster health to return to yellow or green
       ansible.builtin.uri:
         url: "{{ elasticsearch_http_protocol }}://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/health"
         method: GET
         user: elastic
-        password: "{{ elasticstack_password.stdout }}"
+        password: "{{ __elasticstack_password }}"
         validate_certs: no
       register: response
       until: "response.json.status == 'yellow' or response.json.status == 'green'"

roles/elasticsearch/tasks/main.yml

@@ -135,7 +135,7 @@
     - "hostvars[item].inventory_hostname == inventory_hostname"
     - elasticstack_version is defined
     - ansible_facts.packages['elasticsearch'][0].version is defined
-    - elasticstack_password.stdout is defined
+    - __elasticstack_password is defined
     - elasticstack_version is version( ansible_facts.packages['elasticsearch'][0].version, '>')
 
 - name: Install Elasticsearch - rpm - full stack
@@ -147,6 +147,8 @@
   when:
     - ansible_os_family == "RedHat"
     - elasticstack_full_stack | bool
+    # fresh install only, an upgrade goes through elasticsearch-rolling-upgrade.yml
+    - "'elasticsearch' not in ansible_facts.packages"
 
 - name: Install Elasticsearch - rpm - standalone
   ansible.builtin.package:
@@ -155,13 +157,17 @@
   when:
     - ansible_os_family == "RedHat"
     - not elasticstack_full_stack | bool
+    # fresh install only, an upgrade goes through elasticsearch-rolling-upgrade.yml
+    - "'elasticsearch' not in ansible_facts.packages"
 
 - name: Install Elasticsearch - deb
   ansible.builtin.package:
     name: "{{ elasticsearch_package }}"
     state: present
   when:
     - ansible_os_family == "Debian"
+    # fresh install only, an upgrade goes through elasticsearch-rolling-upgrade.yml
+    - "'elasticsearch' not in ansible_facts.packages"
 
 - name: Configure Elasticsearch
   ansible.builtin.template:
@@ -268,7 +274,7 @@
     - name: Check for cluster status without security
       ansible.builtin.uri:
 # kics-scan ignore-line
         url: "http://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
       register: elasticsearch_cluster_status
       ignore_errors: "{{ ansible_check_mode }}"
       until: elasticsearch_cluster_status.json.status == "green"

roles/elasticsearch/templates/elasticsearch.yml.j2

@@ -1,4 +1,3 @@
-# test
 {{ ansible_managed | comment }}
 
 node.name: "{{ elasticsearch_nodename }}"

roles/elasticstack/tasks/elasticstack-passwords.yml

@@ -4,15 +4,23 @@
   ansible.builtin.stat:
     path: "{{ elasticstack_initial_passwords }}"
   delegate_to: "{{ elasticstack_ca_host | default(omit, true) }}"
+  run_once: true
   register: elasticsearch_passwords_file
 
 - name: Fetch Elastic password # noqa: risky-shell-pipe
   ansible.builtin.shell: >
     if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
     grep "PASSWORD elastic" {{ elasticstack_initial_passwords }} |
     awk {' print $4 '}
-  register: elasticstack_password
+  register: __elasticstack_password_fetch
+  run_once: true
   changed_when: false
   no_log: "{{ elasticstack_no_log }}"
   delegate_to: "{{ elasticstack_ca_host | default(omit, true) }}"
   when: elasticsearch_passwords_file.stat.exists | bool
+
+- name: Persist the elastic password as a fact for all roles
+  ansible.builtin.set_fact:
+    __elasticstack_password: "{{ __elasticstack_password_fetch.stdout }}"
+  when: __elasticstack_password_fetch.stdout is defined
+  no_log: "{{ elasticstack_no_log }}"

roles/elasticstack/tasks/main.yml

@@ -10,7 +10,6 @@
 
     - name: Set versions for components
       ansible.builtin.import_tasks: elasticstack-versions.yml
-      when: elasticstack_full_stack | bool
 
     - name: Fetch passwords if passwords are initialized
       ansible.builtin.import_tasks: elasticstack-passwords.yml